diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java index b959f1af9b315..62c0c3bf5a9ba 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java @@ -17,6 +17,7 @@ import com.facebook.presto.hive.HiveType; import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -80,7 +81,7 @@ public class CachingHiveMetastore private final LoadingCache partitionStatisticsCache; private final LoadingCache>> viewNamesCache; private final LoadingCache> partitionCache; - private final LoadingCache>> partitionFilterCache; + private final LoadingCache> partitionFilterCache; private final LoadingCache>> partitionNamesCache; private final LoadingCache> tablePrivilegesCache; private final LoadingCache> rolesCache; @@ -167,7 +168,7 @@ public Map loadAll(Iterable>() @@ -524,17 +525,22 @@ private Optional> loadPartitionNames(HiveTableName hiveTableName) } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public List getPartitionNamesByFilter( + String databaseName, + String tableName, + Map partitionPredicates) { - return get(partitionFilterCache, partitionFilter(databaseName, tableName, parts)); + return get( + partitionFilterCache, + partitionFilter(databaseName, tableName, partitionPredicates)); } - private Optional> loadPartitionNamesByParts(PartitionFilter partitionFilter) + private List loadPartitionNamesByFilter(PartitionFilter partitionFilter) { - return delegate.getPartitionNamesByParts( + return delegate.getPartitionNamesByFilter( partitionFilter.getHiveTableName().getDatabaseName(), partitionFilter.getHiveTableName().getTableName(), - partitionFilter.getParts()); + partitionFilter.getPartitionPredicates()); } @Override diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java index 4207e867c0704..017ba1ec22689 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive.metastore; import com.facebook.presto.hive.HiveType; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -76,7 +77,10 @@ public interface ExtendedHiveMetastore Optional> getPartitionNames(String databaseName, String tableName); - Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts); + List getPartitionNamesByFilter( + String databaseName, + String tableName, + Map partitionPredicates); Map> getPartitionsByNames(String databaseName, String tableName, List partitionNames); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java index 2946984dc970b..bf7202e2e9918 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.TableNotFoundException; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.spi.type.CharType; @@ -42,6 +43,7 @@ import com.facebook.presto.spi.type.VarcharType; import com.google.common.base.CharMatcher; import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -50,6 +52,8 @@ import org.apache.hadoop.hive.metastore.ProtectMode; import org.apache.hadoop.io.Text; import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; import java.io.IOException; import java.math.BigInteger; @@ -60,6 +64,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Properties; @@ -68,6 +73,7 @@ import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.type.Chars.padSpaces; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.base.Strings.padEnd; @@ -102,6 +108,7 @@ public class MetastoreUtil public static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__"; @SuppressWarnings("OctalInteger") public static final FsPermission ALL_PERMISSIONS = new FsPermission((short) 0777); + private static final String PARTITION_VALUE_WILDCARD = ""; private MetastoreUtil() { @@ -186,7 +193,7 @@ public static Properties getHiveSchema( schema.setProperty(BUCKET_COUNT, "0"); } - for (Map.Entry param : storage.getSerdeParameters().entrySet()) { + for (Entry param : storage.getSerdeParameters().entrySet()) { schema.setProperty(param.getKey(), (param.getValue() != null) ? param.getValue() : ""); } schema.setProperty(SERIALIZATION_LIB, storage.getStorageFormat().getSerDe()); @@ -234,7 +241,7 @@ public static Properties getHiveSchema( } if (tableParameters != null) { - for (Map.Entry entry : tableParameters.entrySet()) { + for (Entry entry : tableParameters.entrySet()) { // add non-null parameters to the schema if (entry.getValue() != null) { schema.setProperty(entry.getKey(), entry.getValue()); @@ -248,10 +255,10 @@ public static Properties getHiveSchema( public static Properties getHiveSchema(Map serdeParameters, Map tableParameters) { Properties schema = new Properties(); - for (Map.Entry param : serdeParameters.entrySet()) { + for (Entry param : serdeParameters.entrySet()) { schema.setProperty(param.getKey(), (param.getValue() != null) ? param.getValue() : ""); } - for (Map.Entry entry : tableParameters.entrySet()) { + for (Entry entry : tableParameters.entrySet()) { // add non-null parameters to the schema if (entry.getValue() != null) { schema.setProperty(entry.getKey(), entry.getValue()); @@ -618,4 +625,72 @@ private static String getRenameErrorMessage(Path source, Path target) { return format("Error moving data files from %s to final location %s", source, target); } + + public static List convertPredicateToParts(Map partitionPredicates) + { + List filter = new ArrayList<>(); + + for (Entry partitionPredicate : partitionPredicates.entrySet()) { + Domain domain = partitionPredicate.getValue(); + if (!domain.isAll()) { + if (domain != null && domain.isNullableSingleValue()) { + Object value = domain.getNullableSingleValue(); + Type type = domain.getType(); + filter.add(convertRawValueToString(value, type)); + } + else { + filter.add(PARTITION_VALUE_WILDCARD); + } + } + else { + filter.add(PARTITION_VALUE_WILDCARD); + } + } + + return filter; + } + + public static String convertRawValueToString(Object value, Type type) + { + String val; + if (value == null) { + val = HIVE_DEFAULT_DYNAMIC_PARTITION; + } + else if (type instanceof CharType) { + Slice slice = (Slice) value; + val = padSpaces(slice, type).toStringUtf8(); + } + else if (type instanceof VarcharType) { + Slice slice = (Slice) value; + val = slice.toStringUtf8(); + } + else if (type instanceof DecimalType && !((DecimalType) type).isShort()) { + Slice slice = (Slice) value; + val = Decimals.toString(slice, ((DecimalType) type).getScale()); + } + else if (type instanceof DecimalType && ((DecimalType) type).isShort()) { + val = Decimals.toString((long) value, ((DecimalType) type).getScale()); + } + else if (type instanceof DateType) { + DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.date().withZoneUTC(); + val = dateTimeFormatter.print(TimeUnit.DAYS.toMillis((long) value)); + } + else if (type instanceof TimestampType) { + // we don't have time zone info, so just add a wildcard + val = PARTITION_VALUE_WILDCARD; + } + else if (type instanceof TinyintType + || type instanceof SmallintType + || type instanceof IntegerType + || type instanceof BigintType + || type instanceof DoubleType + || type instanceof RealType + || type instanceof BooleanType) { + val = value.toString(); + } + else { + throw new PrestoException(NOT_SUPPORTED, format("Unsupported partition key type: %s", type.getDisplayName())); + } + return val; + } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java index fa04bb608f7f3..ff999bf41a8fa 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java @@ -13,13 +13,11 @@ */ package com.facebook.presto.hive.metastore; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; +import com.facebook.presto.spi.predicate.Domain; import javax.annotation.concurrent.Immutable; -import java.util.List; +import java.util.Map; import java.util.Objects; import static com.facebook.presto.hive.metastore.HiveTableName.hiveTableName; @@ -30,30 +28,28 @@ public class PartitionFilter { private final HiveTableName hiveTableName; - private final List parts; - @JsonCreator - public PartitionFilter(@JsonProperty("hiveTableName") HiveTableName hiveTableName, @JsonProperty("parts") List parts) + private final Map partitionPredicates; + + public PartitionFilter(HiveTableName hiveTableName, Map partitionPredicates) { this.hiveTableName = requireNonNull(hiveTableName, "hiveTableName is null"); - this.parts = ImmutableList.copyOf(requireNonNull(parts, "parts is null")); + this.partitionPredicates = requireNonNull(partitionPredicates, "effectivePredicate is null"); } - public static PartitionFilter partitionFilter(String databaseName, String tableName, List parts) + public static PartitionFilter partitionFilter(String databaseName, String tableName, Map effectivePredicate) { - return new PartitionFilter(hiveTableName(databaseName, tableName), parts); + return new PartitionFilter(hiveTableName(databaseName, tableName), effectivePredicate); } - @JsonProperty public HiveTableName getHiveTableName() { return hiveTableName; } - @JsonProperty - public List getParts() + public Map getPartitionPredicates() { - return parts; + return partitionPredicates; } @Override @@ -61,7 +57,7 @@ public String toString() { return toStringHelper(this) .add("hiveTableName", hiveTableName) - .add("parts", parts) + .add("partitionPredicates", partitionPredicates) .toString(); } @@ -77,12 +73,12 @@ public boolean equals(Object o) PartitionFilter other = (PartitionFilter) o; return Objects.equals(hiveTableName, other.hiveTableName) && - Objects.equals(parts, other.parts); + Objects.equals(partitionPredicates, other.partitionPredicates); } @Override public int hashCode() { - return Objects.hash(hiveTableName, parts); + return Objects.hash(hiveTableName, partitionPredicates); } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java index 4e04e56be62c5..1e11a955f194c 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java @@ -18,6 +18,7 @@ import com.facebook.presto.hive.HiveType; import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -70,7 +71,7 @@ public class RecordingHiveMetastore private final Cache>> allViewsCache; private final Cache> partitionCache; private final Cache>> partitionNamesCache; - private final Cache>> partitionNamesByPartsCache; + private final Cache> partitionNamesByFilterCache; private final Cache, Map>> partitionsByNamesCache; private final Cache> tablePrivilegesCache; private final Cache> roleGrantsCache; @@ -93,7 +94,7 @@ public RecordingHiveMetastore(@ForRecordingHiveMetastore ExtendedHiveMetastore d allViewsCache = createCache(metastoreClientConfig); partitionCache = createCache(metastoreClientConfig); partitionNamesCache = createCache(metastoreClientConfig); - partitionNamesByPartsCache = createCache(metastoreClientConfig); + partitionNamesByFilterCache = createCache(metastoreClientConfig); partitionsByNamesCache = createCache(metastoreClientConfig); tablePrivilegesCache = createCache(metastoreClientConfig); roleGrantsCache = createCache(metastoreClientConfig); @@ -120,7 +121,7 @@ void loadRecording() allViewsCache.putAll(toMap(recording.getAllViews())); partitionCache.putAll(toMap(recording.getPartitions())); partitionNamesCache.putAll(toMap(recording.getPartitionNames())); - partitionNamesByPartsCache.putAll(toMap(recording.getPartitionNamesByParts())); + partitionNamesByFilterCache.putAll(toMap(recording.getPartitionNamesByFilter())); partitionsByNamesCache.putAll(toMap(recording.getPartitionsByNames())); tablePrivilegesCache.putAll(toMap(recording.getTablePrivileges())); roleGrantsCache.putAll(toMap(recording.getRoleGrants())); @@ -158,7 +159,7 @@ public void writeRecording() toPairs(allViewsCache), toPairs(partitionCache), toPairs(partitionNamesCache), - toPairs(partitionNamesByPartsCache), + toPairs(partitionNamesByFilterCache), toPairs(partitionsByNamesCache), toPairs(tablePrivilegesCache), toPairs(roleGrantsCache)); @@ -343,12 +344,15 @@ public Optional> getPartitionNames(String databaseName, String tabl } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public List getPartitionNamesByFilter( + String databaseName, + String tableName, + Map partitionPredicates) { return loadValue( - partitionNamesByPartsCache, - partitionFilter(databaseName, tableName, parts), - () -> delegate.getPartitionNamesByParts(databaseName, tableName, parts)); + partitionNamesByFilterCache, + partitionFilter(databaseName, tableName, partitionPredicates).toString(), + () -> delegate.getPartitionNamesByFilter(databaseName, tableName, partitionPredicates)); } @Override @@ -493,7 +497,7 @@ public static class Recording private final List>>> allViews; private final List>> partitions; private final List>>> partitionNames; - private final List>>> partitionNamesByParts; + private final List>> partitionNamesByFilter; private final List, Map>>> partitionsByNames; private final List>> tablePrivileges; private final List>> roleGrants; @@ -511,7 +515,7 @@ public Recording( @JsonProperty("allViews") List>>> allViews, @JsonProperty("partitions") List>> partitions, @JsonProperty("partitionNames") List>>> partitionNames, - @JsonProperty("partitionNamesByParts") List>>> partitionNamesByParts, + @JsonProperty("partitionNamesByFilter") List>> partitionNamesByFilter, @JsonProperty("partitionsByNames") List, Map>>> partitionsByNames, @JsonProperty("tablePrivileges") List>> tablePrivileges, @JsonProperty("roleGrants") List>> roleGrants) @@ -527,7 +531,7 @@ public Recording( this.allViews = allViews; this.partitions = partitions; this.partitionNames = partitionNames; - this.partitionNamesByParts = partitionNamesByParts; + this.partitionNamesByFilter = partitionNamesByFilter; this.partitionsByNames = partitionsByNames; this.tablePrivileges = tablePrivileges; this.roleGrants = roleGrants; @@ -600,9 +604,9 @@ public List>>> getPartitionNames() } @JsonProperty - public List>>> getPartitionNamesByParts() + public List>> getPartitionNamesByFilter() { - return partitionNamesByParts; + return partitionNamesByFilter; } @JsonProperty diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java index 3e042cad020b4..676a24511c7de 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java @@ -26,6 +26,7 @@ import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.PrincipalType; import com.facebook.presto.spi.security.RoleGrant; @@ -58,7 +59,6 @@ import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY; @@ -69,6 +69,7 @@ import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME; +import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts; import static com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory; import static com.facebook.presto.hive.metastore.MetastoreUtil.getFileSystem; import static com.facebook.presto.hive.metastore.MetastoreUtil.isPrestoView; @@ -94,6 +95,7 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; import static org.apache.hadoop.hive.common.FileUtils.makePartName; public class SemiTransactionalHiveMetastore @@ -512,16 +514,19 @@ public synchronized void truncateUnpartitionedTable(ConnectorSession session, St public synchronized Optional> getPartitionNames(String databaseName, String tableName) { - return doGetPartitionNames(databaseName, tableName, Optional.empty()); + return doGetPartitionNames(databaseName, tableName, ImmutableMap.of()); } - public synchronized Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public synchronized Optional> getPartitionNamesByFilter(String databaseName, String tableName, Map effectivePredicate) { - return doGetPartitionNames(databaseName, tableName, Optional.of(parts)); + return doGetPartitionNames(databaseName, tableName, effectivePredicate); } @GuardedBy("this") - private Optional> doGetPartitionNames(String databaseName, String tableName, Optional> parts) + private Optional> doGetPartitionNames( + String databaseName, + String tableName, + Map partitionPredicates) { checkHoldsLock(); @@ -538,8 +543,9 @@ private Optional> doGetPartitionNames(String databaseName, String t break; case PRE_EXISTING_TABLE: { Optional> partitionNameResult; - if (parts.isPresent()) { - partitionNameResult = delegate.getPartitionNamesByParts(databaseName, tableName, parts.get()); + List partitionColumns = table.get().getPartitionColumns(); + if (!partitionPredicates.isEmpty()) { + partitionNameResult = Optional.of(delegate.getPartitionNamesByFilter(databaseName, tableName, partitionPredicates)); } else { partitionNameResult = delegate.getPartitionNames(databaseName, tableName); @@ -579,12 +585,14 @@ private Optional> doGetPartitionNames(String databaseName, String t } // add newly-added partitions to the results from underlying metastore if (!partitionActionsOfTable.isEmpty()) { - List columnNames = table.get().getPartitionColumns().stream().map(Column::getName).collect(Collectors.toList()); + List partitionColumns = table.get().getPartitionColumns(); + List partitionColumnNames = partitionColumns.stream().map(Column::getName).collect(toList()); + List parts = convertPredicateToParts(partitionPredicates); for (Action partitionAction : partitionActionsOfTable.values()) { if (partitionAction.getType() == ActionType.ADD) { List values = partitionAction.getData().getPartition().getValues(); - if (!parts.isPresent() || partitionValuesMatch(values, parts.get())) { - resultBuilder.add(makePartName(columnNames, values)); + if (parts.isEmpty() || partitionValuesMatch(values, parts)) { + resultBuilder.add(makePartName(partitionColumnNames, values)); } } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java index 9b2c339842409..bda0fb4f77286 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java @@ -37,6 +37,7 @@ import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.ConnectorIdentity; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; @@ -76,6 +77,7 @@ import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; +import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts; import static com.facebook.presto.hive.metastore.MetastoreUtil.extractPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; @@ -92,6 +94,7 @@ import static com.facebook.presto.spi.security.PrincipalType.USER; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; @@ -893,12 +896,17 @@ public synchronized Optional getPartition(String databaseName, String } @Override - public synchronized Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public synchronized List getPartitionNamesByFilter( + String databaseName, + String tableName, + Map partitionPredicates) { + List parts = convertPredicateToParts(partitionPredicates); // todo this should be more efficient by selectively walking the directory tree return getPartitionNames(databaseName, tableName).map(partitionNames -> partitionNames.stream() .filter(partitionName -> partitionMatches(partitionName, parts)) - .collect(toList())); + .collect(toImmutableList())) + .orElse(ImmutableList.of()); } private static boolean partitionMatches(String partitionName, List parts) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java index 029a90757ef57..7794b54709a06 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java @@ -76,6 +76,7 @@ import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.ConnectorIdentity; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; @@ -106,6 +107,7 @@ import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; +import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts; import static com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; @@ -656,16 +658,20 @@ public Optional> getPartitionNames(String databaseName, String tabl * ['', '2', ''] * * - * @param parts Full or partial list of partition values to filter on. Keys without filter will be empty strings. + * @param partitionPredicates Full or partial list of partition values to filter on. Keys without filter will be empty strings. * @return a list of partition names. */ @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public List getPartitionNamesByFilter( + String databaseName, + String tableName, + Map partitionPredicates) { Table table = getTableOrElseThrow(databaseName, tableName); + List parts = convertPredicateToParts(partitionPredicates); String expression = buildGlueExpression(table.getPartitionColumns(), parts); List partitions = getPartitions(databaseName, tableName, expression); - return Optional.of(buildPartitionNames(table.getPartitionColumns(), partitions)); + return buildPartitionNames(table.getPartitionColumns(), partitions); } private List getPartitions(String databaseName, String tableName, String expression) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java index 9a5aa221c05e7..1bac20bfda3f0 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive.metastore.thrift; import com.facebook.presto.hive.HiveType; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; @@ -27,6 +28,7 @@ import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -252,9 +254,12 @@ public Optional> getPartitionNames(String databaseName, String tabl } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public List getPartitionNamesByFilter( + String databaseName, + String tableName, + Map partitionPredicates) { - return delegate.getPartitionNamesByParts(databaseName, tableName, parts); + return delegate.getPartitionNamesByFilter(databaseName, tableName, partitionPredicates); } @Override diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java index 26a161eb7eafd..753531002f6b1 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java @@ -13,12 +13,14 @@ */ package com.facebook.presto.hive.metastore.thrift; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -68,6 +70,8 @@ public interface HiveMetastore Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts); + List getPartitionNamesByFilter(String databaseName, String tableName, Map partitionPredicates); + Optional getPartition(String databaseName, String tableName, List partitionValues); List getPartitionsByNames(String databaseName, String tableName, List partitionNames); diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java index 3b90b287ae7a2..b41e3e4112003 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -79,6 +80,7 @@ import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_VIEW_FLAG; +import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.createMetastoreColumnStatistics; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiPrincipalType; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiTable; @@ -942,6 +944,13 @@ public Optional> getPartitionNamesByParts(String databaseName, Stri } } + @Override + public List getPartitionNamesByFilter(String databaseName, String tableName, Map partitionPredicates) + { + List parts = convertPredicateToParts(partitionPredicates); + return getPartitionNamesByParts(databaseName, tableName, parts).orElse(ImmutableList.of()); + } + @Override public void addPartitions(String databaseName, String tableName, List partitionsWithStatistics) { diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java index 697dfc25b3981..ea056ae445b54 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java @@ -20,6 +20,7 @@ import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore; import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastoreStats; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListeningExecutorService; import io.airlift.units.Duration; @@ -42,6 +43,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; @Test(singleThreaded = true) public class TestCachingHiveMetastore @@ -149,25 +151,23 @@ public void testInvalidGetPartitionNames() @Test public void testGetPartitionNamesByParts() { - ImmutableList parts = ImmutableList.of(); ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByFilter(TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByFilter(TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); - assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); + assertEquals(metastore.getPartitionNamesByFilter(TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } public void testInvalidGetPartitionNamesByParts() { - ImmutableList parts = ImmutableList.of(); - assertFalse(metastore.getPartitionNamesByParts(BAD_DATABASE, TEST_TABLE, parts).isPresent()); + assertTrue(metastore.getPartitionNamesByFilter(BAD_DATABASE, TEST_TABLE, ImmutableMap.of()).isEmpty()); } @Test diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java index b4ad968a172dd..20a9e6819ddf5 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java @@ -19,6 +19,7 @@ import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; import com.facebook.presto.hive.metastore.SortingColumn.Order; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -31,6 +32,7 @@ import java.io.File; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -39,11 +41,14 @@ import java.util.concurrent.TimeUnit; import static com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics; +import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts; import static com.facebook.presto.hive.metastore.PrestoTableType.OTHER; import static com.facebook.presto.spi.security.PrincipalType.USER; import static com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE; import static com.facebook.presto.spi.statistics.ColumnStatisticType.MIN_VALUE; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; +import static io.airlift.slice.Slices.utf8Slice; import static org.testng.Assert.assertEquals; public class TestRecordingHiveMetastore @@ -133,7 +138,10 @@ private void validateMetadata(ExtendedHiveMetastore hiveMetastore) assertEquals(hiveMetastore.getAllViews("database"), Optional.empty()); assertEquals(hiveMetastore.getPartition("database", "table", ImmutableList.of("value")), Optional.of(PARTITION)); assertEquals(hiveMetastore.getPartitionNames("database", "table"), Optional.of(ImmutableList.of("value"))); - assertEquals(hiveMetastore.getPartitionNamesByParts("database", "table", ImmutableList.of("value")), Optional.of(ImmutableList.of("value"))); + Map map = new HashMap<>(); + Column column = new Column("column", HiveType.HIVE_STRING, Optional.empty()); + map.put(column, Domain.singleValue(VARCHAR, utf8Slice("value"))); + assertEquals(hiveMetastore.getPartitionNamesByFilter("database", "table", map), ImmutableList.of("value")); assertEquals(hiveMetastore.getPartitionsByNames("database", "table", ImmutableList.of("value")), ImmutableMap.of("value", Optional.of(PARTITION))); assertEquals(hiveMetastore.listTablePrivileges("database", "table", new PrestoPrincipal(USER, "user")), ImmutableSet.of(PRIVILEGE_INFO)); assertEquals(hiveMetastore.listRoles(), ImmutableSet.of("role")); @@ -242,13 +250,17 @@ public Optional> getPartitionNames(String databaseName, String tabl } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public List getPartitionNamesByFilter( + String databaseName, + String tableName, + Map partitionPredicates) { + List parts = convertPredicateToParts(partitionPredicates); if (databaseName.equals("database") && tableName.equals("table") && parts.equals(ImmutableList.of("value"))) { - return Optional.of(ImmutableList.of("value")); + return ImmutableList.of("value"); } - return Optional.empty(); + return ImmutableList.of(); } @Override diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java index c4f822bdaee58..14da194b4675a 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive.metastore; import com.facebook.presto.hive.HiveType; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -161,7 +162,10 @@ public Optional> getPartitionNames(String databaseName, String tabl } @Override - public Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public List getPartitionNamesByFilter( + String databaseName, + String tableName, + Map partitionPredicates) { throw new UnsupportedOperationException(); } diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java index 28fd8defa2793..3d8dc5a0c95bb 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.SchemaAlreadyExistsException; import com.facebook.presto.hive.TableAlreadyExistsException; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; @@ -22,6 +23,7 @@ import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -53,6 +55,7 @@ import java.util.function.Function; import static com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics; +import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts; import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiPartition; import static com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY; @@ -363,6 +366,13 @@ public synchronized Optional> getPartitionNamesByParts(String datab .collect(toList())); } + @Override + public List getPartitionNamesByFilter(String databaseName, String tableName, Map partitionPredicates) + { + List parts = convertPredicateToParts(partitionPredicates); + return getPartitionNamesByParts(databaseName, tableName, parts).orElse(ImmutableList.of()); + } + private static boolean partitionMatches(Partition partition, String databaseName, String tableName, List parts) { if (!partition.getDbName().equals(databaseName) || diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java index 39f86a57aa683..84e5836265d2a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.HiveBucketing.HiveBucketFilter; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore; import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.spi.ColumnHandle; @@ -26,18 +27,7 @@ import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.TupleDomain; -import com.facebook.presto.spi.type.BigintType; -import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.spi.type.CharType; -import com.facebook.presto.spi.type.DateType; -import com.facebook.presto.spi.type.DecimalType; -import com.facebook.presto.spi.type.Decimals; -import com.facebook.presto.spi.type.DoubleType; -import com.facebook.presto.spi.type.IntegerType; -import com.facebook.presto.spi.type.RealType; -import com.facebook.presto.spi.type.SmallintType; -import com.facebook.presto.spi.type.TimestampType; -import com.facebook.presto.spi.type.TinyintType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.VarcharType; @@ -46,19 +36,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import io.airlift.slice.Slice; import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.ISODateTimeFormat; import javax.inject.Inject; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.TimeUnit; import static com.facebook.presto.hive.HiveBucketing.getHiveBucketFilter; import static com.facebook.presto.hive.HiveBucketing.getHiveBucketHandle; @@ -69,15 +54,12 @@ import static com.facebook.presto.hive.HiveSessionProperties.shouldIgnoreTableBucketing; import static com.facebook.presto.hive.HiveUtil.getPartitionKeyColumnHandles; import static com.facebook.presto.hive.HiveUtil.parsePartitionValue; -import static com.facebook.presto.hive.metastore.MetastoreUtil.HIVE_DEFAULT_DYNAMIC_PARTITION; import static com.facebook.presto.hive.metastore.MetastoreUtil.extractPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.spi.Constraint.alwaysTrue; -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.facebook.presto.spi.type.Chars.padSpaces; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Predicates.not; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -87,8 +69,6 @@ public class HivePartitionManager { - private static final String PARTITION_VALUE_WILDCARD = ""; - private final DateTimeZone timeZone; private final boolean assumeCanonicalPartitionKeys; private final TypeManager typeManager; @@ -123,10 +103,14 @@ public HivePartitionManager( this.domainCompactionThreshold = domainCompactionThreshold; } - public Iterable getPartitionsIterator(SemiTransactionalHiveMetastore metastore, ConnectorTableHandle tableHandle, Constraint constraint, ConnectorSession session) + public Iterable getPartitionsIterator( + SemiTransactionalHiveMetastore metastore, + ConnectorTableHandle tableHandle, + Constraint constraint, + ConnectorSession session) { HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle; - TupleDomain effectivePredicate = constraint.getSummary(); + TupleDomain effectivePredicateColumnHandles = constraint.getSummary(); SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); Table table = getTable(metastore, tableName, isOfflineDataDebugModeEnabled(session)); @@ -137,12 +121,63 @@ public Iterable getPartitionsIterator(SemiTransactionalHiveMetast .map(column -> typeManager.getType(column.getTypeSignature())) .collect(toList()); - return partitionColumns.isEmpty() ? ImmutableList.of(new HivePartition(tableName)) : () -> getFilteredPartitionNames(metastore, tableName, partitionColumns, effectivePredicate).stream() - // Apply extra filters which could not be done by getFilteredPartitionNames - .map(partitionName -> parseValuesAndFilterPartition(tableName, partitionName, partitionColumns, partitionTypes, constraint)) - .filter(Optional::isPresent) - .map(Optional::get) - .iterator(); + Map effectivePredicate = createPartitionPredicates( + effectivePredicateColumnHandles, + partitionColumns, + assumeCanonicalPartitionKeys); + + if (partitionColumns.isEmpty()) { + return ImmutableList.of(new HivePartition(tableName)); + } + else { + return () -> { + List filteredPartitionNames = getFilteredPartitionNames(metastore, tableName, effectivePredicate); + return filteredPartitionNames.stream() + // Apply extra filters which could not be done by getFilteredPartitionNames + .map(partitionName -> parseValuesAndFilterPartition(tableName, partitionName, partitionColumns, partitionTypes, constraint)) + .filter(Optional::isPresent) + .map(Optional::get) + .iterator(); + }; + } + } + + private Map createPartitionPredicates( + TupleDomain effectivePredicateColumnHandles, + List partitionColumns, + boolean assumeCanonicalPartitionKeys) + { + Optional> domains = effectivePredicateColumnHandles.getDomains(); + if (domains.isPresent()) { + Map columnHandleDomainMap = domains.get(); + ImmutableMap.Builder partitionPredicateBuilder = ImmutableMap.builder(); + for (HiveColumnHandle partitionColumn : partitionColumns) { + Column key = new Column(partitionColumn.getName(), partitionColumn.getHiveType(), partitionColumn.getComment()); + if (columnHandleDomainMap.containsKey(partitionColumn)) { + if (assumeCanonicalPartitionKeys) { + partitionPredicateBuilder.put(key, columnHandleDomainMap.get(partitionColumn)); + } + else { + Type type = typeManager.getType(partitionColumn.getTypeSignature()); + if (type instanceof VarcharType || type instanceof CharType) { + partitionPredicateBuilder.put(key, columnHandleDomainMap.get(partitionColumn)); + } + else { + Domain allDomain = Domain.all(typeManager.getType(partitionColumn.getTypeSignature())); + partitionPredicateBuilder.put(key, allDomain); + } + } + } + else { + Domain allDomain = Domain.all(typeManager.getType(partitionColumn.getTypeSignature())); + partitionPredicateBuilder.put(key, allDomain); + } + } + return partitionPredicateBuilder.build(); + } + else { + return ImmutableMap.of(); + } } public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastore, ConnectorTableHandle tableHandle, Constraint constraint, ConnectorSession session) @@ -316,72 +351,14 @@ private Table getTable(SemiTransactionalHiveMetastore metastore, SchemaTableName return table; } - private List getFilteredPartitionNames(SemiTransactionalHiveMetastore metastore, SchemaTableName tableName, List partitionKeys, TupleDomain effectivePredicate) + private List getFilteredPartitionNames(SemiTransactionalHiveMetastore metastore, SchemaTableName tableName, Map partitionPredicates) { - if (effectivePredicate.isNone()) { + if (partitionPredicates.isEmpty()) { return ImmutableList.of(); } - List filter = new ArrayList<>(); - for (HiveColumnHandle partitionKey : partitionKeys) { - Domain domain = effectivePredicate.getDomains().get().get(partitionKey); - if (domain != null && domain.isNullableSingleValue()) { - Object value = domain.getNullableSingleValue(); - Type type = domain.getType(); - if (value == null) { - filter.add(HIVE_DEFAULT_DYNAMIC_PARTITION); - } - else if (type instanceof CharType) { - Slice slice = (Slice) value; - filter.add(padSpaces(slice, type).toStringUtf8()); - } - else if (type instanceof VarcharType) { - Slice slice = (Slice) value; - filter.add(slice.toStringUtf8()); - } - // Types above this have only a single possible representation for each value. - // Types below this may have multiple representations for a single value. For - // example, a boolean column may represent the false value as "0", "false" or "False". - // The metastore distinguishes between these representations, so we cannot prune partitions - // unless we know that all partition values use the canonical Java representation. - else if (!assumeCanonicalPartitionKeys) { - filter.add(PARTITION_VALUE_WILDCARD); - } - else if (type instanceof DecimalType && !((DecimalType) type).isShort()) { - Slice slice = (Slice) value; - filter.add(Decimals.toString(slice, ((DecimalType) type).getScale())); - } - else if (type instanceof DecimalType && ((DecimalType) type).isShort()) { - filter.add(Decimals.toString((long) value, ((DecimalType) type).getScale())); - } - else if (type instanceof DateType) { - DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.date().withZoneUTC(); - filter.add(dateTimeFormatter.print(TimeUnit.DAYS.toMillis((long) value))); - } - else if (type instanceof TimestampType) { - // we don't have time zone info, so just add a wildcard - filter.add(PARTITION_VALUE_WILDCARD); - } - else if (type instanceof TinyintType - || type instanceof SmallintType - || type instanceof IntegerType - || type instanceof BigintType - || type instanceof DoubleType - || type instanceof RealType - || type instanceof BooleanType) { - filter.add(value.toString()); - } - else { - throw new PrestoException(NOT_SUPPORTED, format("Unsupported partition key type: %s", type.getDisplayName())); - } - } - else { - filter.add(PARTITION_VALUE_WILDCARD); - } - } - // fetch the partition names - return metastore.getPartitionNamesByParts(tableName.getSchemaName(), tableName.getTableName(), filter) + return metastore.getPartitionNamesByFilter(tableName.getSchemaName(), tableName.getTableName(), partitionPredicates) .orElseThrow(() -> new TableNotFoundException(tableName)); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java index af37473cc83b2..a9f3b28e01ab3 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java @@ -15,6 +15,7 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePageSinkMetadata; @@ -31,6 +32,7 @@ import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; @@ -231,7 +233,7 @@ public synchronized Optional> getPartitionNames(String databaseName } @Override - public synchronized Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + public synchronized Optional> getPartitionNamesByFilter(String databaseName, String tableName, Map effectivePredicate) { return Optional.ofNullable(partitionsMap.get(new HiveTableName(databaseName, tableName))); }