diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java index 8a860f8766de6..da842bb440532 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java @@ -14,7 +14,6 @@ package com.facebook.presto.hive.metastore.file; import com.facebook.airlift.json.JsonCodec; -import com.facebook.airlift.log.Logger; import com.facebook.presto.common.predicate.Domain; import com.facebook.presto.common.type.Type; import com.facebook.presto.hive.HdfsContext; @@ -64,7 +63,6 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import javax.annotation.concurrent.ThreadSafe; @@ -100,7 +98,6 @@ import static com.facebook.presto.hive.metastore.MetastoreUtil.extractPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveBasicStatistics; import static com.facebook.presto.hive.metastore.MetastoreUtil.getPartitionNamesWithEmptyVersion; -import static com.facebook.presto.hive.metastore.MetastoreUtil.isIcebergTable; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.updateStatisticsParameters; @@ -130,21 +127,19 @@ public class FileHiveMetastore implements ExtendedHiveMetastore { - private static final Logger LOG = Logger.get(FileHiveMetastore.class); + protected static final String PRESTO_SCHEMA_FILE_NAME = ".prestoSchema"; + protected static final String PRESTO_PERMISSIONS_DIRECTORY_NAME = ".prestoPermissions"; private static final String PUBLIC_ROLE_NAME = "public"; private static final String ADMIN_ROLE_NAME = "admin"; - private static final String PRESTO_SCHEMA_FILE_NAME = ".prestoSchema"; - private static final String PRESTO_PERMISSIONS_DIRECTORY_NAME = ".prestoPermissions"; // todo there should be a way to manage the admins list private static final Set ADMIN_USERS = ImmutableSet.of("admin", "hive", "hdfs"); - private final HdfsEnvironment hdfsEnvironment; - private final Path catalogDirectory; - private final HdfsContext hdfsContext; - private final FileSystem metadataFileSystem; + protected final HdfsEnvironment hdfsEnvironment; + protected final HdfsContext hdfsContext; + protected final FileSystem metadataFileSystem; + private final Path catalogDirectory; private final BiMap lockedHiveTables = HashBiMap.create(); - private long currentLockId; private final JsonCodec databaseCodec = JsonCodec.jsonCodec(DatabaseMetadata.class); private final JsonCodec tableCodec = JsonCodec.jsonCodec(TableMetadata.class); @@ -154,6 +149,8 @@ public class FileHiveMetastore private final JsonCodec> roleGrantsCodec = JsonCodec.listJsonCodec(RoleGrant.class); private final JsonCodec> tableConstraintCodec = JsonCodec.listJsonCodec(TableConstraint.class); + private long currentLockId; + @Inject public FileHiveMetastore(HdfsEnvironment hdfsEnvironment, FileHiveMetastoreConfig config) { @@ -229,19 +226,6 @@ public synchronized Optional getDatabase(MetastoreContext metastoreCon .map(databaseMetadata -> databaseMetadata.toDatabase(databaseName, databaseMetadataDirectory.toString())); } - private Database getRequiredDatabase(MetastoreContext metastoreContext, String databaseName) - { - return getDatabase(metastoreContext, databaseName) - .orElseThrow(() -> new SchemaNotFoundException(databaseName)); - } - - private void verifyDatabaseNotExists(MetastoreContext metastoreContext, String databaseName) - { - if (getDatabase(metastoreContext, databaseName).isPresent()) { - throw new SchemaAlreadyExistsException(databaseName); - } - } - @Override public synchronized List getAllDatabases(MetastoreContext metastoreContext) { @@ -271,14 +255,7 @@ else if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equa } else if (table.getTableType().equals(EXTERNAL_TABLE)) { try { - Path externalLocation = new Path(table.getStorage().getLocation()); - FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); - if (!externalFileSystem.isDirectory(externalLocation)) { - throw new PrestoException(HIVE_METASTORE_ERROR, "External table location does not exist"); - } - if (isChildDirectory(catalogDirectory, externalLocation) && !isIcebergTable(table)) { - throw new PrestoException(HIVE_METASTORE_ERROR, "External table location can not be inside the system metadata directory"); - } + validateExternalLocation(new Path(table.getStorage().getLocation()), catalogDirectory); } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external location", e); @@ -354,19 +331,6 @@ public synchronized Map getPartitionStatistics(Meta return statistics.build(); } - private Table getRequiredTable(MetastoreContext metastoreContext, String databaseName, String tableName) - { - return getTable(metastoreContext, databaseName, tableName) - .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); - } - - private void verifyTableNotExists(MetastoreContext metastoreContext, String newDatabaseName, String newTableName) - { - if (getTable(metastoreContext, newDatabaseName, newTableName).isPresent()) { - throw new TableAlreadyExistsException(new SchemaTableName(newDatabaseName, newTableName)); - } - } - @Override public synchronized void updateTableStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, Function update) { @@ -484,9 +448,7 @@ public synchronized MetastoreOperationResult replaceTable(MetastoreContext metas checkArgument(!newTable.getTableType().equals(TEMPORARY_TABLE), "temporary tables must never be stored in the metastore"); Table table = getRequiredTable(metastoreContext, databaseName, tableName); - if ((!table.getTableType().equals(VIRTUAL_VIEW) || !newTable.getTableType().equals(VIRTUAL_VIEW)) && !isIcebergTable(table)) { - throw new PrestoException(HIVE_METASTORE_ERROR, "Only views can be updated with replaceTable"); - } + validateReplaceTableType(table, newTable); if (!table.getDatabaseName().equals(databaseName) || !table.getTableName().equals(tableName)) { throw new PrestoException(HIVE_METASTORE_ERROR, "Replacement table must have same name"); } @@ -514,89 +476,18 @@ public synchronized MetastoreOperationResult renameTable(MetastoreContext metast requireNonNull(tableName, "tableName is null"); requireNonNull(newDatabaseName, "newDatabaseName is null"); requireNonNull(newTableName, "newTableName is null"); - Table table = getRequiredTable(metastoreContext, databaseName, tableName); getRequiredDatabase(metastoreContext, newDatabaseName); + // verify new table does not exist verifyTableNotExists(metastoreContext, newDatabaseName, newTableName); Path metadataDirectory = getTableMetadataDirectory(databaseName, tableName); Path newMetadataDirectory = getTableMetadataDirectory(newDatabaseName, newTableName); - - if (isIcebergTable(table)) { - renameIcebergTable(metadataDirectory, newMetadataDirectory); - } - else { - renameTable(metadataDirectory, newMetadataDirectory); - } + renameTable(metadataDirectory, newMetadataDirectory); return EMPTY_RESULT; } - private void renameIcebergTable(Path originalMetadataDirectory, Path newMetadataDirectory) - { - Optional rollbackAction = Optional.empty(); - try { - // If the directory `.prestoPermissions` exists, copy it to the new table metadata directory - Path originTablePermissionDir = new Path(originalMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME); - Path newTablePermissionDir = new Path(newMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME); - if (metadataFileSystem.exists(originTablePermissionDir)) { - if (!FileUtil.copy(metadataFileSystem, originTablePermissionDir, - metadataFileSystem, newTablePermissionDir, false, metadataFileSystem.getConf())) { - throw new IOException(format("Could not rename table. Failed to copy directory: %s to %s", originTablePermissionDir, newTablePermissionDir)); - } - else { - rollbackAction = Optional.of(() -> { - try { - metadataFileSystem.delete(newTablePermissionDir, true); - } - catch (IOException e) { - LOG.warn("Could not delete table permission directory: %s", newTablePermissionDir); - } - }); - } - } - - // Rename file `.prestoSchema` to change it to the new metadata path - // This will atomically execute the table renaming behavior - Path originMetadataFile = new Path(originalMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); - Path newMetadataFile = new Path(newMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); - renamePath(originMetadataFile, newMetadataFile, - format("Could not rename table. Failed to rename file %s to %s", originMetadataFile, newMetadataFile)); - - // Subsequent action, delete the redundant directory `.prestoPermissions` from the original table metadata path - try { - metadataFileSystem.delete(new Path(originalMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME), true); - } - catch (IOException e) { - // ignore - } - } - catch (IOException e) { - // If table renaming fails and rollback action has already been recorded, perform the rollback action to clean up junk files - rollbackAction.ifPresent(Runnable::run); - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - private void renameTable(Path originalMetadataDirectory, Path newMetadataDirectory) - { - try { - renamePath(originalMetadataDirectory, newMetadataDirectory, - format("Could not rename table. Failed to rename directory %s to %s", originalMetadataDirectory, newMetadataDirectory)); - } - catch (IOException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - } - - private void renamePath(Path originalPath, Path targetPath, String errorMessage) - throws IOException - { - if (!metadataFileSystem.rename(originalPath, targetPath)) { - throw new IOException(errorMessage); - } - } - @Override public synchronized MetastoreOperationResult addColumn(MetastoreContext metastoreContext, String databaseName, String tableName, String columnName, HiveType columnType, String columnComment) { @@ -670,23 +561,6 @@ public synchronized MetastoreOperationResult dropColumn(MetastoreContext metasto return EMPTY_RESULT; } - private void alterTable(String databaseName, String tableName, Function alterFunction) - { - requireNonNull(databaseName, "databaseName is null"); - requireNonNull(tableName, "tableName is null"); - - Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName); - - TableMetadata oldTableSchema = readSchemaFile("table", tableMetadataDirectory, tableCodec) - .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); - TableMetadata newTableSchema = alterFunction.apply(oldTableSchema); - if (oldTableSchema == newTableSchema) { - return; - } - - writeSchemaFile("table", tableMetadataDirectory, tableCodec, newTableSchema, true); - } - @Override public synchronized MetastoreOperationResult addPartitions(MetastoreContext metastoreContext, String databaseName, String tableName, List partitions) { @@ -741,35 +615,6 @@ public synchronized MetastoreOperationResult addPartitions(MetastoreContext meta } } - private void verifiedPartition(Table table, Partition partition) - { - Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); - - if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equals(MATERIALIZED_VIEW)) { - if (!partitionMetadataDirectory.equals(new Path(partition.getStorage().getLocation()))) { - throw new PrestoException(HIVE_METASTORE_ERROR, "Partition directory must be " + partitionMetadataDirectory); - } - } - else if (table.getTableType().equals(EXTERNAL_TABLE)) { - try { - Path externalLocation = new Path(partition.getStorage().getLocation()); - FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); - if (!externalFileSystem.isDirectory(externalLocation)) { - throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location does not exist"); - } - if (isChildDirectory(catalogDirectory, externalLocation)) { - throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location can not be inside the system metadata directory"); - } - } - catch (IOException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external partition location", e); - } - } - else { - throw new PrestoException(NOT_SUPPORTED, "Partitions can not be added to " + table.getTableType()); - } - } - @Override public synchronized void dropPartition(MetastoreContext metastoreContext, String databaseName, String tableName, List partitionValues, boolean deleteData) { @@ -906,49 +751,6 @@ public synchronized Set listRoleGrants(MetastoreContext metastoreCont return result.build(); } - private synchronized Set listRoleGrantsSanitized(MetastoreContext metastoreContext) - { - Set grants = readRoleGrantsFile(); - Set existingRoles = listRoles(metastoreContext); - return removeDuplicatedEntries(removeNonExistingRoles(grants, existingRoles)); - } - - private Set removeDuplicatedEntries(Set grants) - { - Map map = new HashMap<>(); - for (RoleGrant grant : grants) { - RoleGranteeTuple tuple = new RoleGranteeTuple(grant.getRoleName(), grant.getGrantee()); - map.merge(tuple, grant, (first, second) -> first.isGrantable() ? first : second); - } - return ImmutableSet.copyOf(map.values()); - } - - private static Set removeNonExistingRoles(Set grants, Set existingRoles) - { - ImmutableSet.Builder result = ImmutableSet.builder(); - for (RoleGrant grant : grants) { - if (!existingRoles.contains(grant.getRoleName())) { - continue; - } - PrestoPrincipal grantee = grant.getGrantee(); - if (grantee.getType() == ROLE && !existingRoles.contains(grantee.getName())) { - continue; - } - result.add(grant); - } - return result.build(); - } - - private Set readRoleGrantsFile() - { - return ImmutableSet.copyOf(readFile("roleGrants", getRoleGrantsFile(), roleGrantsCodec).orElse(ImmutableList.of())); - } - - private void writeRoleGrantsFile(Set roleGrants) - { - writeFile("roleGrants", getRoleGrantsFile(), roleGrantsCodec, ImmutableList.copyOf(roleGrants), true); - } - @Override public synchronized Optional> getPartitionNames(MetastoreContext metastoreContext, String databaseName, String tableName) { @@ -972,45 +774,6 @@ public synchronized Optional> getPartitionNames(M return Optional.of(getPartitionNamesWithEmptyVersion(partitionNames)); } - private List> listPartitions(Path director, List partitionColumns) - { - if (partitionColumns.isEmpty()) { - return ImmutableList.of(); - } - - try { - String directoryPrefix = partitionColumns.get(0).getName() + '='; - - List> partitionValues = new ArrayList<>(); - for (FileStatus fileStatus : metadataFileSystem.listStatus(director)) { - if (!fileStatus.isDirectory()) { - continue; - } - if (!fileStatus.getPath().getName().startsWith(directoryPrefix)) { - continue; - } - - List> childPartitionValues; - if (partitionColumns.size() == 1) { - childPartitionValues = ImmutableList.of(new ArrayDeque<>()); - } - else { - childPartitionValues = listPartitions(fileStatus.getPath(), partitionColumns.subList(1, partitionColumns.size())); - } - - String value = unescapePathName(fileStatus.getPath().getName().substring(directoryPrefix.length())); - for (ArrayDeque childPartition : childPartitionValues) { - childPartition.addFirst(value); - partitionValues.add(childPartition); - } - } - return partitionValues; - } - catch (IOException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, "Error listing partition directories", e); - } - } - @Override public synchronized Optional getPartition(MetastoreContext metastoreContext, String databaseName, String tableName, List partitionValues) { @@ -1054,21 +817,6 @@ public List getPartitionNamesWithVersionByFilter( throw new UnsupportedOperationException(); } - private static boolean partitionMatches(String partitionName, List parts) - { - List values = toPartitionValues(partitionName); - if (values.size() != parts.size()) { - return false; - } - for (int i = 0; i < values.size(); i++) { - String part = parts.get(i); - if (!part.isEmpty() && !values.get(i).equals(part)) { - return false; - } - } - return true; - } - @Override public synchronized Map> getPartitionsByNames(MetastoreContext metastoreContext, String databaseName, String tableName, List partitionNameWithVersion) { @@ -1171,6 +919,7 @@ else if (tableConstraint instanceof NotNullConstraint) { return EMPTY_RESULT; } + @Override public List> getTableConstraints(MetastoreContext metastoreContext, String schemaName, String tableName) { Set rawConstraints = readConstraintsFile(schemaName, tableName); @@ -1221,6 +970,213 @@ public synchronized void unlock(MetastoreContext metastoreContext, long lockId) lockedHiveTables.remove(lockId); } + protected void validateExternalLocation(Path externalLocation, Path catalogDirectory) + throws IOException + { + FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); + if (!externalFileSystem.isDirectory(externalLocation)) { + throw new PrestoException(HIVE_METASTORE_ERROR, "External table location does not exist"); + } + if (isChildDirectory(catalogDirectory, externalLocation)) { + throw new PrestoException(HIVE_METASTORE_ERROR, "External table location can not be inside the system metadata directory"); + } + } + + protected void validateReplaceTableType(Table originTable, Table newTable) + { + if (!originTable.getTableType().equals(VIRTUAL_VIEW) || !newTable.getTableType().equals(VIRTUAL_VIEW)) { + throw new PrestoException(HIVE_METASTORE_ERROR, "Only views can be updated with replaceTable"); + } + } + + protected void renameTable(Path originalMetadataDirectory, Path newMetadataDirectory) + { + try { + renamePath(originalMetadataDirectory, newMetadataDirectory, + format("Could not rename table. Failed to rename directory %s to %s", originalMetadataDirectory, newMetadataDirectory)); + } + catch (IOException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } + + protected void renamePath(Path originalPath, Path targetPath, String errorMessage) + throws IOException + { + if (!metadataFileSystem.rename(originalPath, targetPath)) { + throw new IOException(errorMessage); + } + } + + private Table getRequiredTable(MetastoreContext metastoreContext, String databaseName, String tableName) + { + return getTable(metastoreContext, databaseName, tableName) + .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); + } + + private void verifyTableNotExists(MetastoreContext metastoreContext, String newDatabaseName, String newTableName) + { + if (getTable(metastoreContext, newDatabaseName, newTableName).isPresent()) { + throw new TableAlreadyExistsException(new SchemaTableName(newDatabaseName, newTableName)); + } + } + + private void alterTable(String databaseName, String tableName, Function alterFunction) + { + requireNonNull(databaseName, "databaseName is null"); + requireNonNull(tableName, "tableName is null"); + + Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName); + + TableMetadata oldTableSchema = readSchemaFile("table", tableMetadataDirectory, tableCodec) + .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); + TableMetadata newTableSchema = alterFunction.apply(oldTableSchema); + if (oldTableSchema == newTableSchema) { + return; + } + + writeSchemaFile("table", tableMetadataDirectory, tableCodec, newTableSchema, true); + } + + private void verifiedPartition(Table table, Partition partition) + { + Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); + + if (table.getTableType().equals(MANAGED_TABLE) || table.getTableType().equals(MATERIALIZED_VIEW)) { + if (!partitionMetadataDirectory.equals(new Path(partition.getStorage().getLocation()))) { + throw new PrestoException(HIVE_METASTORE_ERROR, "Partition directory must be " + partitionMetadataDirectory); + } + } + else if (table.getTableType().equals(EXTERNAL_TABLE)) { + try { + Path externalLocation = new Path(partition.getStorage().getLocation()); + FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); + if (!externalFileSystem.isDirectory(externalLocation)) { + throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location does not exist"); + } + if (isChildDirectory(catalogDirectory, externalLocation)) { + throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location can not be inside the system metadata directory"); + } + } + catch (IOException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external partition location", e); + } + } + else { + throw new PrestoException(NOT_SUPPORTED, "Partitions can not be added to " + table.getTableType()); + } + } + + private synchronized Set listRoleGrantsSanitized(MetastoreContext metastoreContext) + { + Set grants = readRoleGrantsFile(); + Set existingRoles = listRoles(metastoreContext); + return removeDuplicatedEntries(removeNonExistingRoles(grants, existingRoles)); + } + + private Set removeDuplicatedEntries(Set grants) + { + Map map = new HashMap<>(); + for (RoleGrant grant : grants) { + RoleGranteeTuple tuple = new RoleGranteeTuple(grant.getRoleName(), grant.getGrantee()); + map.merge(tuple, grant, (first, second) -> first.isGrantable() ? first : second); + } + return ImmutableSet.copyOf(map.values()); + } + + private static Set removeNonExistingRoles(Set grants, Set existingRoles) + { + ImmutableSet.Builder result = ImmutableSet.builder(); + for (RoleGrant grant : grants) { + if (!existingRoles.contains(grant.getRoleName())) { + continue; + } + PrestoPrincipal grantee = grant.getGrantee(); + if (grantee.getType() == ROLE && !existingRoles.contains(grantee.getName())) { + continue; + } + result.add(grant); + } + return result.build(); + } + + private Set readRoleGrantsFile() + { + return ImmutableSet.copyOf(readFile("roleGrants", getRoleGrantsFile(), roleGrantsCodec).orElse(ImmutableList.of())); + } + + private void writeRoleGrantsFile(Set roleGrants) + { + writeFile("roleGrants", getRoleGrantsFile(), roleGrantsCodec, ImmutableList.copyOf(roleGrants), true); + } + + private List> listPartitions(Path director, List partitionColumns) + { + if (partitionColumns.isEmpty()) { + return ImmutableList.of(); + } + + try { + String directoryPrefix = partitionColumns.get(0).getName() + '='; + + List> partitionValues = new ArrayList<>(); + for (FileStatus fileStatus : metadataFileSystem.listStatus(director)) { + if (!fileStatus.isDirectory()) { + continue; + } + if (!fileStatus.getPath().getName().startsWith(directoryPrefix)) { + continue; + } + + List> childPartitionValues; + if (partitionColumns.size() == 1) { + childPartitionValues = ImmutableList.of(new ArrayDeque<>()); + } + else { + childPartitionValues = listPartitions(fileStatus.getPath(), partitionColumns.subList(1, partitionColumns.size())); + } + + String value = unescapePathName(fileStatus.getPath().getName().substring(directoryPrefix.length())); + for (ArrayDeque childPartition : childPartitionValues) { + childPartition.addFirst(value); + partitionValues.add(childPartition); + } + } + return partitionValues; + } + catch (IOException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, "Error listing partition directories", e); + } + } + + private static boolean partitionMatches(String partitionName, List parts) + { + List values = toPartitionValues(partitionName); + if (values.size() != parts.size()) { + return false; + } + for (int i = 0; i < values.size(); i++) { + String part = parts.get(i); + if (!part.isEmpty() && !values.get(i).equals(part)) { + return false; + } + } + return true; + } + + private Database getRequiredDatabase(MetastoreContext metastoreContext, String databaseName) + { + return getDatabase(metastoreContext, databaseName) + .orElseThrow(() -> new SchemaNotFoundException(databaseName)); + } + + private void verifyDatabaseNotExists(MetastoreContext metastoreContext, String databaseName) + { + if (getDatabase(metastoreContext, databaseName).isPresent()) { + throw new SchemaAlreadyExistsException(databaseName); + } + } + private synchronized void setTablePrivileges( MetastoreContext metastoreContext, PrestoPrincipal grantee, diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java index 3663d1b9086db..b0392dd4e2bb4 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java @@ -18,11 +18,11 @@ import com.facebook.presto.hive.PartitionMutator; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HiveMetastoreCacheStats; -import com.facebook.presto.hive.metastore.HiveMetastoreModule; import com.facebook.presto.hive.metastore.HivePartitionMutator; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; import com.facebook.presto.hive.metastore.MetastoreCacheStats; import com.facebook.presto.hive.metastore.MetastoreConfig; +import com.facebook.presto.iceberg.hive.IcebergHiveMetastoreModule; import com.google.inject.Binder; import com.google.inject.Scopes; @@ -48,7 +48,7 @@ public IcebergHiveModule(String connectorId, Optional met @Override public void setup(Binder binder) { - install(new HiveMetastoreModule(this.connectorId, this.metastore)); + install(new IcebergHiveMetastoreModule(this.connectorId, this.metastore)); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); configBinder(binder).bindConfig(IcebergHiveTableOperationsConfig.class); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergFileHiveMetastore.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergFileHiveMetastore.java new file mode 100644 index 0000000000000..1822a92b0bd19 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergFileHiveMetastore.java @@ -0,0 +1,114 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.hive; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.hive.HdfsEnvironment; +import com.facebook.presto.hive.metastore.Table; +import com.facebook.presto.hive.metastore.file.FileHiveMetastore; +import com.facebook.presto.hive.metastore.file.FileHiveMetastoreConfig; +import com.facebook.presto.spi.PrestoException; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; + +import javax.annotation.concurrent.ThreadSafe; +import javax.inject.Inject; + +import java.io.IOException; +import java.util.Optional; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; +import static java.lang.String.format; + +@ThreadSafe +public class IcebergFileHiveMetastore + extends FileHiveMetastore +{ + private static final Logger LOG = Logger.get(IcebergFileHiveMetastore.class); + + @Inject + public IcebergFileHiveMetastore(HdfsEnvironment hdfsEnvironment, FileHiveMetastoreConfig config) + { + this(hdfsEnvironment, config.getCatalogDirectory(), config.getMetastoreUser()); + } + + public IcebergFileHiveMetastore(HdfsEnvironment hdfsEnvironment, String catalogDirectory, String metastoreUser) + { + super(hdfsEnvironment, catalogDirectory, metastoreUser); + } + + @Override + protected void validateExternalLocation(Path externalLocation, Path catalogDirectory) + throws IOException + { + FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); + if (!externalFileSystem.isDirectory(externalLocation)) { + throw new PrestoException(HIVE_METASTORE_ERROR, "External table location does not exist"); + } + } + + @Override + protected void validateReplaceTableType(Table originTable, Table newTable) + {} + + @Override + protected void renameTable(Path originalMetadataDirectory, Path newMetadataDirectory) + { + Optional rollbackAction = Optional.empty(); + try { + // If the directory `.prestoPermissions` exists, copy it to the new table metadata directory + Path originTablePermissionDir = new Path(originalMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME); + Path newTablePermissionDir = new Path(newMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME); + if (metadataFileSystem.exists(originTablePermissionDir)) { + if (!FileUtil.copy(metadataFileSystem, originTablePermissionDir, + metadataFileSystem, newTablePermissionDir, false, metadataFileSystem.getConf())) { + throw new IOException(format("Could not rename table. Failed to copy directory: %s to %s", originTablePermissionDir, newTablePermissionDir)); + } + else { + rollbackAction = Optional.of(() -> { + try { + metadataFileSystem.delete(newTablePermissionDir, true); + } + catch (IOException e) { + // Ignore the exception and print a warn level log + LOG.warn("Could not delete table permission directory: %s", newTablePermissionDir); + } + }); + } + } + + // Rename file `.prestoSchema` to change it to the new metadata path + // This will atomically execute the table renaming behavior + Path originMetadataFile = new Path(originalMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); + Path newMetadataFile = new Path(newMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); + renamePath(originMetadataFile, newMetadataFile, + format("Could not rename table. Failed to rename file %s to %s", originMetadataFile, newMetadataFile)); + + // Subsequent action, delete the redundant directory `.prestoPermissions` from the original table metadata path + try { + metadataFileSystem.delete(new Path(originalMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME), true); + } + catch (IOException e) { + // Ignore the exception and print a warn level log + LOG.warn("Could not delete table permission directory: %s", originalMetadataDirectory); + } + } + catch (IOException e) { + // If table renaming fails and rollback action has already been recorded, perform the rollback action to clean up junk files + rollbackAction.ifPresent(Runnable::run); + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveFileMetastoreModule.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveFileMetastoreModule.java new file mode 100644 index 0000000000000..f06a2fba1542e --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveFileMetastoreModule.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.hive; + +import com.facebook.presto.hive.ForCachingHiveMetastore; +import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; +import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; +import com.facebook.presto.hive.metastore.file.FileHiveMetastoreConfig; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static java.util.Objects.requireNonNull; +import static org.weakref.jmx.ObjectNames.generatedNameOf; +import static org.weakref.jmx.guice.ExportBinder.newExporter; + +public class IcebergHiveFileMetastoreModule + implements Module +{ + private final String connectorId; + + public IcebergHiveFileMetastoreModule(String connectorId) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); + } + + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(FileHiveMetastoreConfig.class); + binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(IcebergFileHiveMetastore.class).in(Scopes.SINGLETON); + binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); + newExporter(binder).export(ExtendedHiveMetastore.class) + .as(generatedNameOf(InMemoryCachingHiveMetastore.class, connectorId)); + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveMetastoreModule.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveMetastoreModule.java new file mode 100644 index 0000000000000..daad609172ff2 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveMetastoreModule.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.hive; + +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; +import com.facebook.presto.hive.metastore.MetastoreConfig; +import com.facebook.presto.hive.metastore.glue.GlueMetastoreModule; +import com.facebook.presto.hive.metastore.thrift.ThriftMetastoreModule; +import com.google.inject.Binder; +import com.google.inject.Module; + +import java.util.Optional; + +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; + +public class IcebergHiveMetastoreModule + extends AbstractConfigurationAwareModule +{ + private final String connectorId; + private final Optional metastore; + + public IcebergHiveMetastoreModule(String connectorId, Optional metastore) + { + this.connectorId = connectorId; + this.metastore = metastore; + } + + @Override + protected void setup(Binder binder) + { + if (metastore.isPresent()) { + binder.bind(ExtendedHiveMetastore.class).toInstance(metastore.get()); + } + else { + bindMetastoreModule("thrift", new ThriftMetastoreModule(connectorId)); + bindMetastoreModule("file", new IcebergHiveFileMetastoreModule(connectorId)); + bindMetastoreModule("glue", new GlueMetastoreModule(connectorId)); + } + } + + private void bindMetastoreModule(String name, Module module) + { + install(installModuleIf( + MetastoreConfig.class, + metastore -> name.equalsIgnoreCase(metastore.getMetastoreType()), + module)); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java index 08bf8ae9eeb72..b567f45566fb9 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java @@ -28,7 +28,7 @@ import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.MetastoreContext; -import com.facebook.presto.hive.metastore.file.FileHiveMetastore; +import com.facebook.presto.iceberg.hive.IcebergFileHiveMetastore; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.tests.DistributedQueryRunner; @@ -250,7 +250,7 @@ private static ExtendedHiveMetastore getFileHiveMetastore(Path dataDirectory) MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of(), hiveClientConfig); HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); - return new FileHiveMetastore(hdfsEnvironment, dataDirectory.toFile().toURI().toString(), "test"); + return new IcebergFileHiveMetastore(hdfsEnvironment, dataDirectory.toFile().toURI().toString(), "test"); } public static Path getIcebergDataDirectoryPath(Path dataDirectory, String catalogType, FileFormat format, boolean addStorageFormatToPath) diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java index 3954524b959b2..f0f8627813bf7 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergDistributedHive.java @@ -14,7 +14,6 @@ package com.facebook.presto.iceberg.hive; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; -import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.iceberg.IcebergDistributedTestBase; import com.facebook.presto.iceberg.IcebergHiveTableOperationsConfig; import com.facebook.presto.iceberg.IcebergUtil; @@ -82,7 +81,7 @@ protected Table loadTable(String tableName) protected ExtendedHiveMetastore getFileHiveMetastore() { - FileHiveMetastore fileHiveMetastore = new FileHiveMetastore(getHdfsEnvironment(), + IcebergFileHiveMetastore fileHiveMetastore = new IcebergFileHiveMetastore(getHdfsEnvironment(), getCatalogDirectory().getPath(), "test"); return memoizeMetastore(fileHiveMetastore, false, 1000, 0); diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergHiveStatistics.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergHiveStatistics.java index 9bbac609739e1..b4df3315d0467 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergHiveStatistics.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergHiveStatistics.java @@ -30,7 +30,6 @@ import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; -import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.iceberg.CatalogType; import com.facebook.presto.iceberg.IcebergColumnHandle; import com.facebook.presto.iceberg.IcebergHiveTableOperationsConfig; @@ -579,7 +578,7 @@ private Table loadTable(String tableName) protected ExtendedHiveMetastore getFileHiveMetastore() { - FileHiveMetastore fileHiveMetastore = new FileHiveMetastore(getHdfsEnvironment(), + IcebergFileHiveMetastore fileHiveMetastore = new IcebergFileHiveMetastore(getHdfsEnvironment(), Optional.of(getCatalogDirectory(HIVE)) .filter(File::exists) .map(File::getPath) diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java index d83095286dfa7..406d522ecfc20 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergSmokeHive.java @@ -21,7 +21,6 @@ import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; -import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.iceberg.IcebergConfig; import com.facebook.presto.iceberg.IcebergDistributedSmokeTestBase; import com.facebook.presto.iceberg.IcebergHiveTableOperationsConfig; @@ -68,7 +67,7 @@ protected static HdfsEnvironment getHdfsEnvironment() protected ExtendedHiveMetastore getFileHiveMetastore() { - FileHiveMetastore fileHiveMetastore = new FileHiveMetastore(getHdfsEnvironment(), + IcebergFileHiveMetastore fileHiveMetastore = new IcebergFileHiveMetastore(getHdfsEnvironment(), getCatalogDirectory().toFile().getPath(), "test"); return memoizeMetastore(fileHiveMetastore, false, 1000, 0); diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestRenameTableOnFragileFileSystem.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestRenameTableOnFragileFileSystem.java index 4a6ea4b7cd284..4b04b327d82d7 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestRenameTableOnFragileFileSystem.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestRenameTableOnFragileFileSystem.java @@ -33,7 +33,6 @@ import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.file.DatabaseMetadata; -import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.hive.metastore.file.FileHiveMetastoreConfig; import com.facebook.presto.hive.metastore.file.TableMetadata; import com.facebook.presto.iceberg.CommitTaskData; @@ -347,7 +346,7 @@ private void testRenameTableWithFailSignalAndValidation(FailSignal failSignal, R { FileHiveMetastoreConfig config = createFileHiveMetastoreConfig(); TestingHdfsEnvironment hdfsEnvironment = getTestingHdfsEnvironment(); - FileHiveMetastore metastore = new FileHiveMetastore(hdfsEnvironment, config); + IcebergFileHiveMetastore metastore = new IcebergFileHiveMetastore(hdfsEnvironment, config); IcebergHiveMetadata icebergHiveMetadata = (IcebergHiveMetadata) getIcebergHiveMetadata(metastore); ExtendedFileSystem fileSystem = hdfsEnvironment.getFileSystem(connectorSession.getUser(), new Path(originSchemaMetadataPath), new Configuration()); try { diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/procedure/TestIcebergRegisterAndUnregisterProcedure.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/procedure/TestIcebergRegisterAndUnregisterProcedure.java index 2d2603e881028..67bd536ac481a 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/procedure/TestIcebergRegisterAndUnregisterProcedure.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/procedure/TestIcebergRegisterAndUnregisterProcedure.java @@ -24,9 +24,9 @@ import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.MetastoreContext; -import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.iceberg.IcebergConfig; import com.facebook.presto.iceberg.IcebergPlugin; +import com.facebook.presto.iceberg.hive.IcebergFileHiveMetastore; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.testing.QueryRunner; @@ -540,7 +540,7 @@ protected static HdfsEnvironment getHdfsEnvironment() protected ExtendedHiveMetastore getFileHiveMetastore() { - FileHiveMetastore fileHiveMetastore = new FileHiveMetastore(getHdfsEnvironment(), + IcebergFileHiveMetastore fileHiveMetastore = new IcebergFileHiveMetastore(getHdfsEnvironment(), getCatalogDirectory().toFile().getPath(), "test"); return memoizeMetastore(fileHiveMetastore, false, 1000, 0); diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/procedure/TestRemoveOrphanFilesProcedureHive.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/procedure/TestRemoveOrphanFilesProcedureHive.java index 51b95f0947cee..d1f20a56494b8 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/procedure/TestRemoveOrphanFilesProcedureHive.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/procedure/TestRemoveOrphanFilesProcedureHive.java @@ -17,10 +17,10 @@ import com.facebook.presto.hive.HiveColumnConverterProvider; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.MetastoreContext; -import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.iceberg.HiveTableOperations; import com.facebook.presto.iceberg.IcebergHiveTableOperationsConfig; import com.facebook.presto.iceberg.IcebergUtil; +import com.facebook.presto.iceberg.hive.IcebergFileHiveMetastore; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; @@ -108,7 +108,7 @@ void dropTableFromCatalog(String tableName) private ExtendedHiveMetastore getFileHiveMetastore() { - FileHiveMetastore fileHiveMetastore = new FileHiveMetastore(getHdfsEnvironment(), + IcebergFileHiveMetastore fileHiveMetastore = new IcebergFileHiveMetastore(getHdfsEnvironment(), getCatalogDirectory(HIVE).getPath(), "test"); return memoizeMetastore(fileHiveMetastore, false, 1000, 0);