diff --git a/pom.xml b/pom.xml index 78a5921257f82..012bc850b7d9e 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ 1.6.8 6.2.1 1.9.17 + 2.1.1 + org.alluxio:alluxio-shaded-client org.codehaus.plexus:plexus-utils com.google.guava:guava diff --git a/presto-hive-metastore/pom.xml b/presto-hive-metastore/pom.xml index 72fdeb75fdd43..49b6566790ef0 100644 --- a/presto-hive-metastore/pom.xml +++ b/presto-hive-metastore/pom.xml @@ -16,6 +16,11 @@ + + org.alluxio + alluxio-shaded-client + + com.facebook.presto presto-spi diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java index 972fdc150e739..0bb21a3019368 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive.metastore; import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.presto.hive.metastore.alluxio.AlluxioMetastoreModule; import com.facebook.presto.hive.metastore.file.FileMetastoreModule; import com.facebook.presto.hive.metastore.glue.GlueMetastoreModule; import com.facebook.presto.hive.metastore.thrift.ThriftMetastoreModule; @@ -46,6 +47,7 @@ protected void setup(Binder binder) bindMetastoreModule("thrift", new ThriftMetastoreModule(connectorId)); bindMetastoreModule("file", new FileMetastoreModule(connectorId)); bindMetastoreModule("glue", new GlueMetastoreModule(connectorId)); + bindMetastoreModule("alluxio", new AlluxioMetastoreModule()); } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioHiveMetastore.java new file mode 100644 index 0000000000000..5b26561495bf5 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioHiveMetastore.java @@ -0,0 +1,379 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore.alluxio; + +import alluxio.client.table.TableMasterClient; +import alluxio.exception.status.AlluxioStatusException; +import alluxio.grpc.table.Constraint; +import alluxio.grpc.table.layout.hive.PartitionInfo; +import com.facebook.presto.hive.HiveBasicStatistics; +import com.facebook.presto.hive.HiveType; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.Database; +import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; +import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PartitionStatistics; +import com.facebook.presto.hive.metastore.PartitionWithStatistics; +import com.facebook.presto.hive.metastore.PrincipalPrivileges; +import com.facebook.presto.hive.metastore.Table; +import com.facebook.presto.hive.metastore.thrift.HiveMetastore; +import com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil; +import com.facebook.presto.spi.NotFoundException; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.predicate.Domain; +import com.facebook.presto.spi.security.PrestoPrincipal; +import com.facebook.presto.spi.security.RoleGrant; +import com.facebook.presto.spi.statistics.ColumnStatisticType; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; + +/** + * Implementation of the {@link HiveMetastore} interface through Alluxio. + */ +public class AlluxioHiveMetastore + implements ExtendedHiveMetastore +{ + private final TableMasterClient client; + + @Inject + public AlluxioHiveMetastore(TableMasterClient client) + { + this.client = requireNonNull(client, "client is null"); + } + + @Override + public Optional getDatabase(String databaseName) + { + try { + return Optional.of(AlluxioProtoUtils.fromProto(client.getDatabase(databaseName))); + } + catch (AlluxioStatusException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public List getAllDatabases() + { + try { + return client.getAllDatabases(); + } + catch (AlluxioStatusException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional getTable(String databaseName, String tableName) + { + try { + return Optional.of(AlluxioProtoUtils.fromProto(client.getTable(databaseName, tableName))); + } + catch (NotFoundException e) { + return Optional.empty(); + } + catch (AlluxioStatusException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Set getSupportedColumnStatistics(Type type) + { + throw new UnsupportedOperationException("getSupportedColumnStatistics is not supported in AlluxioHiveMetastore"); + } + + @Override + public PartitionStatistics getTableStatistics(String databaseName, String tableName) + { + try { + Table table = getTable(databaseName, tableName).orElseThrow(() -> new PrestoException( + HIVE_METASTORE_ERROR, + String.format("Could not retrieve table %s.%s", databaseName, tableName))); + HiveBasicStatistics basicStats = ThriftMetastoreUtil.getHiveBasicStatistics(table.getParameters()); + // TODO implement logic to populate Map + return new PartitionStatistics(basicStats, ImmutableMap.of()); + } + catch (Exception e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Map getPartitionStatistics(String databaseName, String tableName, Set partitionNames) + { + // TODO implement partition statistics + // currently returns a map of partitionName to empty statistics to satisfy presto requirements + return partitionNames.stream().collect(toImmutableMap(identity(), (p) -> PartitionStatistics.empty())); + } + + @Override + public void updateTableStatistics(String databaseName, String tableName, Function update) + { + throw new UnsupportedOperationException("updateTableStatistics is not supported in AlluxioHiveMetastore"); + } + + @Override + public void updatePartitionStatistics(String databaseName, String tableName, String partitionName, Function update) + { + throw new UnsupportedOperationException("updatePartitionStatistics is not supported in AlluxioHiveMetastore"); + } + + @Override + public Optional> getAllTables(String databaseName) + { + try { + return Optional.of(client.getAllTables(databaseName)); + } + catch (AlluxioStatusException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Optional> getAllViews(String databaseName) + { + // TODO: Add views on the server side + return Optional.of(Collections.emptyList()); + } + + @Override + public void createDatabase(Database database) + { + throw new UnsupportedOperationException("createDatabase is not supported in AlluxioHiveMetastore"); + } + + @Override + public void dropDatabase(String databaseName) + { + throw new UnsupportedOperationException("dropDatabase is not supported in AlluxioHiveMetastore"); + } + + @Override + public void renameDatabase(String databaseName, String newDatabaseName) + { + throw new UnsupportedOperationException("renameDatabase is not supported in AlluxioHiveMetastore"); + } + + @Override + public void createTable(Table table, PrincipalPrivileges principalPrivileges) + { + throw new UnsupportedOperationException("createTable is not supported in AlluxioHiveMetastore"); + } + + @Override + public void dropTable(String databaseName, String tableName, boolean deleteData) + { + throw new UnsupportedOperationException("dropTable is not supported in AlluxioHiveMetastore"); + } + + @Override + public void replaceTable(String databaseName, String tableName, Table newTable, PrincipalPrivileges principalPrivileges) + { + throw new UnsupportedOperationException("replaceTable is not supported in AlluxioHiveMetastore"); + } + + @Override + public void renameTable(String databaseName, String tableName, String newDatabaseName, String newTableName) + { + throw new UnsupportedOperationException("renameTable is not supported in AlluxioHiveMetastore"); + } + + @Override + public void addColumn(String databaseName, String tableName, String columnName, HiveType columnType, String columnComment) + { + throw new UnsupportedOperationException("addColumn is not supported in AlluxioHiveMetastore"); + } + + @Override + public void renameColumn(String databaseName, String tableName, String oldColumnName, String newColumnName) + { + throw new UnsupportedOperationException("renameColumn is not supported in AlluxioHiveMetastore"); + } + + @Override + public void dropColumn(String databaseName, String tableName, String columnName) + { + throw new UnsupportedOperationException("dropColumn is not supported in AlluxioHiveMetastore"); + } + + @Override + public Optional getPartition(String databaseName, String tableName, List partitionValues) + { + throw new UnsupportedOperationException("getPartition is not supported in AlluxioHiveMetastore"); + } + + @Override + public Optional> getPartitionNames(String databaseName, String tableName) + { + throw new UnsupportedOperationException("getPartitionNames is not supported in AlluxioHiveMetastore"); + } + + @Override + public List getPartitionNamesByFilter(String databaseName, String tableName, Map partitionPredicates) + { + List parts = convertPredicateToParts(partitionPredicates); + return getPartitionNamesByParts(databaseName, tableName, parts).orElse(ImmutableList.of()); + } + + /** + * return a list of partition names by which the values of each partition is at least + * contained which the {@code parts} argument + * + * @param databaseName database name + * @param tableName table name + * @param parts list of values which returned partitions should contain + * @return optionally, a list of strings where each entry is in the form of {key}={value} + */ + public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + { + try { + List partitionInfos = AlluxioProtoUtils.toPartitionInfoList(client.readTable(databaseName, tableName, Constraint.getDefaultInstance())); + + // TODO also check for database name equality + partitionInfos = partitionInfos.stream().filter(p -> p.getTableName().equals(tableName)) + // Filter out any partitions which have values that don't match + .filter(partition -> { + List values = partition.getValuesList(); + if (values.size() != parts.size()) { + return false; + } + for (int i = 0; i < values.size(); i++) { + String constraintPart = parts.get(i); + if (!constraintPart.isEmpty() && !values.get(i).equals(constraintPart)) { + return false; + } + } + return true; + }) + .collect(toImmutableList()); + List partitionNames = partitionInfos.stream().map(PartitionInfo::getPartitionName).collect(toImmutableList()); + return Optional.of(partitionNames); + } + catch (AlluxioStatusException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public Map> getPartitionsByNames(String databaseName, String tableName, List partitionNames) + { + if (partitionNames.isEmpty()) { + return ImmutableMap.of(); + } + + try { + // Get all partitions + List partitionInfos = AlluxioProtoUtils.toPartitionInfoList(client.readTable(databaseName, tableName, Constraint.getDefaultInstance())); + + // TODO also check for database name equality + partitionInfos = partitionInfos.stream().filter(p -> p.getTableName().equals(tableName)).collect(toImmutableList()); + return partitionInfos.stream() + .filter(p -> partitionNames.stream().anyMatch(p.getPartitionName()::equals)) + .collect(toImmutableMap(PartitionInfo::getPartitionName, partitionInfo -> Optional.of(AlluxioProtoUtils.fromProto(partitionInfo)))); + } + catch (AlluxioStatusException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + @Override + public void addPartitions(String databaseName, String tableName, List partitions) + { + throw new UnsupportedOperationException("addPartitions is not supported in AlluxioHiveMetastore"); + } + + @Override + public void dropPartition(String databaseName, String tableName, List parts, boolean deleteData) + { + throw new UnsupportedOperationException("dropPartition is not supported in AlluxioHiveMetastore"); + } + + @Override + public void alterPartition(String databaseName, String tableName, PartitionWithStatistics partition) + { + throw new UnsupportedOperationException("alterPartition is not supported in AlluxioHiveMetastore"); + } + + @Override + public void createRole(String role, String grantor) + { + throw new UnsupportedOperationException("createRole is not supported in AlluxioHiveMetastore"); + } + + @Override + public void dropRole(String role) + { + throw new UnsupportedOperationException("dropRole is not supported in AlluxioHiveMetastore"); + } + + @Override + public Set listRoles() + { + throw new UnsupportedOperationException("listRoles is not supported in AlluxioHiveMetastore"); + } + + @Override + public void grantRoles(Set roles, Set grantees, boolean withAdminOption, PrestoPrincipal grantor) + { + throw new UnsupportedOperationException("grantRoles is not supported in AlluxioHiveMetastore"); + } + + @Override + public void revokeRoles(Set roles, Set grantees, boolean adminOptionFor, PrestoPrincipal grantor) + { + throw new UnsupportedOperationException("revokeRoles is not supported in AlluxioHiveMetastore"); + } + + @Override + public Set listRoleGrants(PrestoPrincipal principal) + { + throw new UnsupportedOperationException("listRoleGrants is not supported in AlluxioHiveMetastore"); + } + + @Override + public void grantTablePrivileges(String databaseName, String tableName, PrestoPrincipal grantee, Set privileges) + { + throw new UnsupportedOperationException("grantTablePrivileges is not supported in AlluxioHiveMetastore"); + } + + @Override + public void revokeTablePrivileges(String databaseName, String tableName, PrestoPrincipal grantee, Set privileges) + { + throw new UnsupportedOperationException("revokeTablePrivileges is not supported in AlluxioHiveMetastore"); + } + + @Override + public Set listTablePrivileges(String databaseName, String tableName, PrestoPrincipal principal) + { + throw new UnsupportedOperationException("listTablePrivileges is not supported in AlluxioHiveMetastore"); + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioHiveMetastoreConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioHiveMetastoreConfig.java new file mode 100644 index 0000000000000..c624d8e94cd27 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioHiveMetastoreConfig.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore.alluxio; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; + +/** + * Configuration for the Alluxio compatible hive metastore interface. + */ +public class AlluxioHiveMetastoreConfig +{ + private String masterAddress; + + public String getMasterAddress() + { + return masterAddress; + } + + @Config("hive.metastore.alluxio.master.address") + @ConfigDescription("Alluxio master address") + public AlluxioHiveMetastoreConfig setMasterAddress(String masterAddress) + { + this.masterAddress = masterAddress; + return this; + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioMetastoreModule.java new file mode 100644 index 0000000000000..ccbc24833cdb9 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioMetastoreModule.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore.alluxio; + +import alluxio.ClientContext; +import alluxio.client.table.RetryHandlingTableMasterClient; +import alluxio.client.table.TableMasterClient; +import alluxio.conf.InstancedConfiguration; +import alluxio.conf.PropertyKey; +import alluxio.master.MasterClientContext; +import alluxio.util.ConfigurationUtils; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; +import com.facebook.presto.hive.metastore.thrift.HiveMetastore; +import com.google.inject.Binder; +import com.google.inject.Inject; +import com.google.inject.Provides; +import com.google.inject.Scopes; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static org.weakref.jmx.ObjectNames.generatedNameOf; +import static org.weakref.jmx.guice.ExportBinder.newExporter; + +/** + * Module for an Alluxio metastore implementation of the {@link HiveMetastore} interface. + */ +public class AlluxioMetastoreModule + extends AbstractConfigurationAwareModule +{ + public AlluxioMetastoreModule() + { + } + + @Override + protected void setup(Binder binder) + { + configBinder(binder).bindConfig(AlluxioHiveMetastoreConfig.class); + + binder.bind(ExtendedHiveMetastore.class).to(AlluxioHiveMetastore.class).in(Scopes.SINGLETON); + newExporter(binder).export(ExtendedHiveMetastore.class).as(generatedNameOf(AlluxioHiveMetastore.class)); + } + + @Provides + @Inject + TableMasterClient provideCatalogMasterClient(AlluxioHiveMetastoreConfig config) + { + InstancedConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults()); + String address = config.getMasterAddress(); + String[] parts = address.split(":", 2); + conf.set(PropertyKey.MASTER_HOSTNAME, parts[0]); + if (parts.length > 1) { + conf.set(PropertyKey.MASTER_RPC_PORT, parts[1]); + } + MasterClientContext context = MasterClientContext.newBuilder(ClientContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()))).build(); + return new RetryHandlingTableMasterClient(context); + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioProtoUtils.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioProtoUtils.java new file mode 100644 index 0000000000000..4f05fccc00858 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/alluxio/AlluxioProtoUtils.java @@ -0,0 +1,160 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore.alluxio; + +import alluxio.grpc.table.FieldSchema; +import alluxio.grpc.table.Layout; +import alluxio.grpc.table.layout.hive.PartitionInfo; +import alluxio.shaded.client.com.google.protobuf.InvalidProtocolBufferException; +import com.facebook.presto.hive.HiveBucketProperty; +import com.facebook.presto.hive.HiveType; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.Database; +import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PrestoTableType; +import com.facebook.presto.hive.metastore.SortingColumn; +import com.facebook.presto.hive.metastore.StorageFormat; +import com.facebook.presto.hive.metastore.Table; +import com.facebook.presto.spi.security.PrincipalType; +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +public class AlluxioProtoUtils +{ + private AlluxioProtoUtils() {} + + public static com.facebook.presto.hive.metastore.Database fromProto(alluxio.grpc.table.Database database) + { + return Database.builder() + .setDatabaseName(database.getDbName()) + .setLocation(database.hasLocation() ? Optional.of(database.getLocation()) : Optional.empty()) + .setOwnerName("") // owner name not yet supported by alluxio + .setOwnerType(PrincipalType.USER) // owner type not yet supported by alluxio + .build(); + } + + public static Table fromProto(alluxio.grpc.table.TableInfo table) + { + if (!table.hasLayout()) { + throw new UnsupportedOperationException("Unsupported table metadata. missing layout."); + } + Layout layout = table.getLayout(); + if (!alluxio.table.ProtoUtils.isHiveLayout(layout)) { + throw new UnsupportedOperationException("Unsupported table layout: " + layout); + } + + try { + PartitionInfo partitionInfo = alluxio.table.ProtoUtils.toHiveLayout(layout); + + // compute the data columns + Set partitionColumns = table.getPartitionColsList().stream().map(FieldSchema::getName).collect(toImmutableSet()); + List dataColumns = table.getSchema().getColsList().stream().filter((field) -> !partitionColumns.contains(field.getName())).collect(toImmutableList()); + + Table.Builder builder = Table.builder() + .setDatabaseName(table.getDbName()) + .setTableName(table.getTableName()) + .setOwner(table.getOwner()) + // TODO - Alluxio should return the actual table type + .setTableType(PrestoTableType.OTHER) + .setDataColumns(dataColumns.stream().map(AlluxioProtoUtils::fromProto).collect(toImmutableList())) + .setPartitionColumns(table.getPartitionColsList().stream().map(AlluxioProtoUtils::fromProto).collect(toImmutableList())) + .setParameters(table.getParametersMap()) + .setViewOriginalText(Optional.empty()) + .setViewExpandedText(Optional.empty()); + alluxio.grpc.table.layout.hive.Storage storage = partitionInfo.getStorage(); + builder.getStorageBuilder() + .setSkewed(storage.getSkewed()) + .setStorageFormat(fromProto(storage.getStorageFormat())) + .setLocation(storage.getLocation()) + .setBucketProperty(storage.hasBucketProperty() ? fromProto(storage.getBucketProperty()) : Optional.empty()) + .setSerdeParameters(storage.getStorageFormat().getSerdelibParametersMap()); + return builder.build(); + } + catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException("Failed to extract PartitionInfo from TableInfo", e); + } + } + + private static SortingColumn fromProto(alluxio.grpc.table.layout.hive.SortingColumn column) + { + if (column.getOrder().equals(alluxio.grpc.table.layout.hive.SortingColumn.SortingOrder.ASCENDING)) { + return new SortingColumn(column.getColumnName(), SortingColumn.Order.ASCENDING); + } + if (column.getOrder().equals(alluxio.grpc.table.layout.hive.SortingColumn.SortingOrder.DESCENDING)) { + return new SortingColumn(column.getColumnName(), SortingColumn.Order.DESCENDING); + } + throw new IllegalArgumentException("Invalid sort order: " + column.getOrder()); + } + + private static Optional fromProto(alluxio.grpc.table.layout.hive.HiveBucketProperty property) + { + // must return empty if buckets <= 0 + if (!property.hasBucketCount() || property.getBucketCount() <= 0) { + return Optional.empty(); + } + List sortedBy = property.getSortedByList().stream().map(AlluxioProtoUtils::fromProto).collect(toImmutableList()); + return Optional.of(new HiveBucketProperty(property.getBucketedByList(), (int) property.getBucketCount(), sortedBy)); + } + + private static Column fromProto(alluxio.grpc.table.FieldSchema column) + { + Optional comment = column.hasComment() ? Optional.of(column.getComment()) : Optional.empty(); + return new Column(column.getName(), HiveType.valueOf(column.getType()), comment); + } + + public static Partition fromProto(alluxio.grpc.table.layout.hive.PartitionInfo info) + { + Partition.Builder builder = Partition.builder() + .setColumns(info.getDataColsList().stream().map(AlluxioProtoUtils::fromProto).collect(toImmutableList())) + .setDatabaseName(info.getDbName()) + .setParameters(info.getParametersMap()) + .setValues(Lists.newArrayList(info.getValuesList())) + .setTableName(info.getTableName()); + + builder.getStorageBuilder() + .setSkewed(info.getStorage().getSkewed()) + .setStorageFormat(fromProto(info.getStorage().getStorageFormat())) + .setLocation(info.getStorage().getLocation()) + .setBucketProperty(info.getStorage().hasBucketProperty() ? fromProto(info.getStorage().getBucketProperty()) : Optional.empty()) + .setSerdeParameters(info.getStorage().getStorageFormat().getSerdelibParametersMap()); + + return builder.build(); + } + + static StorageFormat fromProto(alluxio.grpc.table.layout.hive.StorageFormat format) + { + return StorageFormat.create(format.getSerde(), format.getInputFormat(), format.getOutputFormat()); + } + + public static alluxio.grpc.table.layout.hive.PartitionInfo toPartitionInfo(alluxio.grpc.table.Partition part) + { + try { + return alluxio.table.ProtoUtils.extractHiveLayout(part); + } + catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException("Failed to extract PartitionInfo", e); + } + } + + public static List toPartitionInfoList(List parts) + { + return parts.stream().map(AlluxioProtoUtils::toPartitionInfo).collect(toImmutableList()); + } +} diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/alluxio/TestAlluxioHiveMetastoreConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/alluxio/TestAlluxioHiveMetastoreConfig.java new file mode 100644 index 0000000000000..bd0260236646c --- /dev/null +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/alluxio/TestAlluxioHiveMetastoreConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore.alluxio; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestAlluxioHiveMetastoreConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(AlluxioHiveMetastoreConfig.class) + .setMasterAddress(null)); + } + + @Test + public void testExplicitPropertyMapping() + { + Map properties = new ImmutableMap.Builder() + .put("hive.metastore.alluxio.master.address", "localhost:19998") + .build(); + + AlluxioHiveMetastoreConfig expected = new AlluxioHiveMetastoreConfig() + .setMasterAddress("localhost:19998"); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-server/src/main/assembly/presto.xml b/presto-server/src/main/assembly/presto.xml index 7df731e7e30e6..c7b5995e17068 100644 --- a/presto-server/src/main/assembly/presto.xml +++ b/presto-server/src/main/assembly/presto.xml @@ -11,7 +11,7 @@ NOTICE - +