Skip to content

Allow multiple or missing Hive bucket files #16456

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
highker merged 3 commits into from
Aug 9, 2021
Merged

Allow multiple or missing Hive bucket files #16456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
81fd1f5
Allow multiple or missing Hive bucket files
v-jizhang Jul 21, 2021
6079159
Skip empty bucket creation by default
v-jizhang Jul 21, 2021
8edb15b
Document Hive create empty buckets config
v-jizhang Jul 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions presto-docs/src/main/sphinx/connector/hive.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ Property Name Description

``hive.immutable-partitions`` Can new data be inserted into existing partitions? ``false``

``hive.create-empty-bucket-files`` Should empty files be created for buckets that have no data? ``false``

``hive.max-partitions-per-writers`` Maximum number of partitions per writer. 100

``hive.max-partitions-per-scan`` Maximum number of partitions for a single table scan. 100,000
Expand Down
8 changes: 8 additions & 0 deletions presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
*/
package com.facebook.presto.hive.metastore;

import com.facebook.presto.spi.SchemaTableName;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -77,6 +79,12 @@ public String getTableName()
return tableName;
}

@JsonIgnore
public SchemaTableName getSchemaTableName()
{
return new SchemaTableName(databaseName, tableName);
}

@JsonProperty
public List<String> getValues()
{
Expand Down
95 changes: 43 additions & 52 deletions ...tore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
*/
package com.facebook.presto.hive.metastore;

import com.facebook.presto.spi.SchemaTableName;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -83,6 +85,12 @@ public String getTableName()
return tableName;
}

@JsonIgnore
public SchemaTableName getSchemaTableName()
{
return new SchemaTableName(databaseName, tableName);
}

@JsonProperty
public String getOwner()
{
Expand Down
4 changes: 1 addition & 3 deletions presto-hive/src/main/java/com/facebook/presto/hive/CachingDirectoryLister.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,15 +84,13 @@ public Iterator<HiveFileInfo> list(
PathFilter pathFilter,
HiveDirectoryContext hiveDirectoryContext)
{
SchemaTableName schemaTableName = new SchemaTableName(table.getDatabaseName(), table.getTableName());

List<HiveFileInfo> files = cache.getIfPresent(path);
if (files != null) {
return files.iterator();
}

Iterator<HiveFileInfo> iterator = delegate.list(fileSystem, table, path, namenodeStats, pathFilter, hiveDirectoryContext);
if (hiveDirectoryContext.isCacheable() && cachedTableChecker.isCachedTable(schemaTableName)) {
if (hiveDirectoryContext.isCacheable() && cachedTableChecker.isCachedTable(table.getSchemaTableName())) {
return cachingIterator(iterator, path);
}
return iterator;
Expand Down
14 changes: 14 additions & 0 deletions presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ public class HiveClientConfig
private HiveCompressionCodec orcCompressionCodec = HiveCompressionCodec.GZIP;
private boolean respectTableFormat = true;
private boolean immutablePartitions;
private boolean createEmptyBucketFiles;
private boolean insertOverwriteImmutablePartitions;
private boolean failFastOnInsertIntoImmutablePartitionsEnabled = true;
private int maxPartitionsPerWriter = 100;
Expand Down Expand Up @@ -584,6 +585,19 @@ public HiveClientConfig setImmutablePartitions(boolean immutablePartitions)
return this;
}

public boolean isCreateEmptyBucketFiles()
{
return createEmptyBucketFiles;
}

@Config("hive.create-empty-bucket-files")
@ConfigDescription("Create empty files for buckets that have no data")
public HiveClientConfig setCreateEmptyBucketFiles(boolean createEmptyBucketFiles)
{
this.createEmptyBucketFiles = createEmptyBucketFiles;
return this;
}

public boolean isFailFastOnInsertIntoImmutablePartitionsEnabled()
{
return failFastOnInsertIntoImmutablePartitionsEnabled;
Expand Down
2 changes: 0 additions & 2 deletions presto-hive/src/main/java/com/facebook/presto/hive/HiveInsertTableHandle.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ public HiveInsertTableHandle(
@JsonProperty("schemaName") String schemaName,
@JsonProperty("tableName") String tableName,
@JsonProperty("inputColumns") List<HiveColumnHandle> inputColumns,
@JsonProperty("filePrefix") String filePrefix,
@JsonProperty("pageSinkMetadata") HivePageSinkMetadata pageSinkMetadata,
@JsonProperty("locationHandle") LocationHandle locationHandle,
@JsonProperty("bucketProperty") Optional<HiveBucketProperty> bucketProperty,
Expand All @@ -46,7 +45,6 @@ public HiveInsertTableHandle(
schemaName,
tableName,
inputColumns,
filePrefix,
pageSinkMetadata,
locationHandle,
bucketProperty,
Expand Down
Loading