diff --git a/presto-hive-hadoop2/src/test/java/io/prestosql/plugin/hive/TestHiveAlluxioMetastore.java b/presto-hive-hadoop2/src/test/java/io/prestosql/plugin/hive/TestHiveAlluxioMetastore.java index bf9144481b6c..35c3aad7121b 100644 --- a/presto-hive-hadoop2/src/test/java/io/prestosql/plugin/hive/TestHiveAlluxioMetastore.java +++ b/presto-hive-hadoop2/src/test/java/io/prestosql/plugin/hive/TestHiveAlluxioMetastore.java @@ -309,4 +309,10 @@ public void testViewCreation() { // Alluxio metastore does not support create operations } + + @Override + public void testCreateEmptyTableStaging() + { + // Alluxio metastore does not support create operations + } } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveLocationService.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveLocationService.java index c1f79a9171ea..797acfea176f 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveLocationService.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveLocationService.java @@ -53,7 +53,7 @@ public HiveLocationService(HdfsEnvironment hdfsEnvironment) } @Override - public LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, String schemaName, String tableName, Optional externalLocation) + public LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, String schemaName, String tableName, Optional externalLocation, boolean withData) { HdfsContext context = new HdfsContext(session, schemaName, tableName); Path targetPath = externalLocation.orElseGet(() -> getTableDefaultLocation(context, metastore, hdfsEnvironment, schemaName, tableName)); @@ -64,7 +64,7 @@ public LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, Conn } // TODO detect when existing table's location is a on a different file system than the temporary directory - if (shouldUseTemporaryDirectory(session, context, targetPath, externalLocation)) { + if (withData && shouldUseTemporaryDirectory(session, context, targetPath, externalLocation)) { Path writePath = createTemporaryPath(session, context, hdfsEnvironment, targetPath); return new LocationHandle(targetPath, writePath, false, STAGE_AND_MOVE_TO_TARGET_DIRECTORY); } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java index 716ca448c274..2f13b441d9bc 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java @@ -877,7 +877,7 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe } else { external = false; - LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, Optional.empty()); + LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, Optional.empty(), false); targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); } @@ -1325,7 +1325,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto .collect(toList()); checkPartitionTypesSupported(partitionColumns); - LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, externalLocation); + LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, externalLocation, true); boolean transactional = isTransactional(tableMetadata.getProperties()).orElse(false); AcidTransaction transaction = transactional ? forCreateTable() : NO_ACID_TRANSACTION; diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/LocationService.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/LocationService.java index 224c702b07fa..18eda1eef8d4 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/LocationService.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/LocationService.java @@ -25,7 +25,7 @@ public interface LocationService { - LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, String schemaName, String tableName, Optional externalLocation); + LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, String schemaName, String tableName, Optional externalLocation, boolean withData); LocationHandle forExistingTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table); diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java index 24d0dc4b97f0..a48f8f10ca0b 100644 --- a/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java +++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java @@ -247,6 +247,7 @@ import static io.prestosql.plugin.hive.util.HiveUtil.toPartitionValues; import static io.prestosql.plugin.hive.util.HiveWriteUtils.createDirectory; import static io.prestosql.plugin.hive.util.HiveWriteUtils.getTableDefaultLocation; +import static io.prestosql.plugin.hive.util.HiveWriteUtils.pathExists; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; import static io.prestosql.spi.StandardErrorCode.TRANSACTION_CONFLICT; import static io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; @@ -2207,7 +2208,7 @@ public void testTableCreationIgnoreExisting() try { try (Transaction transaction = newTransaction()) { LocationService locationService = getLocationService(); - LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(), session, schemaName, tableName, Optional.empty()); + LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(), session, schemaName, tableName, Optional.empty(), false); targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); Table table = createSimpleTable(schemaTableName, columns, session, targetPath, "q1"); transaction.getMetastore() @@ -2539,6 +2540,59 @@ public void testEmptyTableCreation() } } + @Test + public void testCreateEmptyTableStaging() + { + for (HiveStorageFormat storageFormat : createTableFormats) { + SchemaTableName temporaryCreateEmptyTable = temporaryTable("create_empty_staging"); + try { + List columns = ImmutableList.of(new Column("test", HIVE_STRING, Optional.empty())); + try (Transaction transaction = newTransaction()) { + ConnectorSession session = newSession(); + String tableOwner = session.getUser(); + String schemaName = temporaryCreateEmptyTable.getSchemaName(); + String tableName = temporaryCreateEmptyTable.getTableName(); + LocationService locationService = getLocationService(); + LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(), + session, + schemaName, + tableName, + Optional.empty(), + false); + Path writePath = locationService.getQueryWriteInfo(locationHandle).getWritePath(); + Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); + Table.Builder tableBuilder = Table.builder() + .setDatabaseName(schemaName) + .setTableName(tableName) + .setOwner(tableOwner) + .setTableType(TableType.MANAGED_TABLE.name()) + .setParameters(ImmutableMap.of( + PRESTO_VERSION_NAME, TEST_SERVER_VERSION, + PRESTO_QUERY_ID_NAME, session.getQueryId())) + .setDataColumns(columns) + .setPartitionColumns(ImmutableList.of()); + tableBuilder.getStorageBuilder() + .setLocation(targetPath.toString()) + .setStorageFormat(StorageFormat.create(storageFormat.getSerDe(), storageFormat.getInputFormat(), storageFormat.getOutputFormat())) + .setBucketProperty(Optional.empty()) + .setSerdeParameters(ImmutableMap.of()); + PrincipalPrivileges principalPrivileges = testingPrincipalPrivilege(tableOwner, session.getUser()); + transaction.getMetastore().createTable(session, tableBuilder.build(), principalPrivileges, Optional.empty(), true, EMPTY_TABLE_STATISTICS); + transaction.commit(); + + HdfsContext context = new HdfsContext(session, schemaName, tableName); + if (locationHandle.getWriteMode() == STAGE_AND_MOVE_TO_TARGET_DIRECTORY) { + assertFalse(pathExists(context, hdfsEnvironment, writePath), + "temp staging directory " + writePath.toString() + " exists"); + } + } + } + finally { + dropTable(temporaryCreateEmptyTable); + } + } + } + @Test public void testViewCreation() { @@ -2783,7 +2837,7 @@ public void testIllegalStorageFormatDuringTableScan() String tableOwner = session.getUser(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); - LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(), session, schemaName, tableName, Optional.empty()); + LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(), session, schemaName, tableName, Optional.empty(), false); Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); //create table whose storage format is null Table.Builder tableBuilder = Table.builder() @@ -4730,7 +4784,7 @@ private void createEmptyTable(SchemaTableName schemaTableName, HiveStorageFormat String tableName = schemaTableName.getTableName(); LocationService locationService = getLocationService(); - LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(), session, schemaName, tableName, Optional.empty()); + LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(), session, schemaName, tableName, Optional.empty(), false); targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); Table.Builder tableBuilder = Table.builder()