Skip to content

Implement efficient way to fetch partition name and values #16129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
highker merged 1 commit into from
May 22, 2021
Merged

Implement efficient way to fetch partition name and values #16129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 23 additions & 8 deletions presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.metastore.ProtectMode;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.io.Text;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormatter;
Expand Down Expand Up @@ -116,7 +115,6 @@
import static org.apache.hadoop.hive.common.FileUtils.unescapePathName;
import static org.apache.hadoop.hive.metastore.ColumnType.typeToThriftType;
import static org.apache.hadoop.hive.metastore.ProtectMode.getProtectModeFromString;
import static org.apache.hadoop.hive.metastore.Warehouse.makeSpecFromName;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_COUNT;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_FIELD_NAME;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT;
Expand Down Expand Up @@ -466,15 +464,32 @@ public static void renameFile(FileSystem fileSystem, Path source, Path target)
}
}

// TODO: https://github.com/prestodb/presto/issues/15974
public static Map<String, String> toPartitionNamesAndValues(String partitionName)
{
try {
return ImmutableMap.copyOf(makeSpecFromName(partitionName));
}
catch (MetaException e) {
throw new PrestoException(HIVE_INVALID_PARTITION_VALUE, "Invalid partition name: " + partitionName);
ImmutableMap.Builder<String, String> resultBuilder = ImmutableMap.builder();
int index = 0;
int length = partitionName.length();

while (index < length) {
int keyStart = index;
while (index < length && partitionName.charAt(index) != '=') {
index++;
}
checkState(index < length, "Invalid partition spec: " + partitionName);

int keyEnd = index++;
int valueStart = index;
while (index < length && partitionName.charAt(index) != '/') {
index++;
}
int valueEnd = index++;

String key = unescapePathName(partitionName.substring(keyStart, keyEnd));
String value = unescapePathName(partitionName.substring(valueStart, valueEnd));
resultBuilder.put(key, value);
}

return resultBuilder.build();
}

public static List<String> toPartitionValues(String partitionName)
Expand Down
50 changes: 50 additions & 0 deletions presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,17 @@
import java.util.AbstractList;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;

import static com.facebook.airlift.testing.Assertions.assertInstanceOf;
import static com.facebook.presto.hive.HiveUtil.getDeserializer;
import static com.facebook.presto.hive.HiveUtil.parseHiveTimestamp;
import static com.facebook.presto.hive.HiveUtil.shouldUseRecordReaderFromInputFormat;
import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionNamesAndValues;
import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_CLASS;
import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT;
import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB;
Expand Down Expand Up @@ -94,6 +97,30 @@ public void testToPartitionValues()
assertToPartitionValues("pk=__HIVE_DEFAULT_PARTITION__");
}

@Test
public void testToPartitionNamesAndValues()
throws MetaException
{
List<String> expectedKeyList1 = new ArrayList<>();
expectedKeyList1.add("ds");
expectedKeyList1.add("event_type");
assertToPartitionNamesAndValues("ds=2015-12-30/event_type=QueryCompletion", expectedKeyList1);

List<String> expectedKeyList2 = new ArrayList<>();
expectedKeyList2.add("a");
expectedKeyList2.add("b");
expectedKeyList2.add("c");
assertToPartitionNamesAndValues("a=1/b=2/c=3", expectedKeyList2);

List<String> expectedKeyList3 = new ArrayList<>();
expectedKeyList3.add("pk");
assertToPartitionNamesAndValues("pk=!@%23$%25%5E&%2A()%2F%3D", expectedKeyList3);

List<String> expectedKeyList4 = new ArrayList<>();
expectedKeyList4.add("pk");
assertToPartitionNamesAndValues("pk=__HIVE_DEFAULT_PARTITION__", expectedKeyList4);
}

@Test
public void testShouldUseRecordReaderFromInputFormat()
{
Expand All @@ -116,6 +143,29 @@ private static void assertToPartitionValues(String partitionName)
assertEquals(actual, expected);
}

private static void assertToPartitionNamesAndValues(String partitionName, List<String> expectedKeyList)
throws MetaException
{
Map<String, String> actual = toPartitionNamesAndValues(partitionName);
AbstractList<String> expectedValueList = new ArrayList<>();
for (String s : expectedKeyList) {
expectedValueList.add(null);
}
Warehouse.makeValsFromName(partitionName, expectedValueList);
checkState(actual.keySet().size() == expectedKeyList.size(), "Keyset size is not same");

for (int index = 0; index < expectedKeyList.size(); index++) {
String key = expectedKeyList.get(index);
if (!actual.containsKey(key)) {
break;
}
checkState(actual.containsKey(key), "Actual result does not contains the key");
String actualValue = actual.get(key);
String expectedValue = expectedValueList.get(index);
checkState(actualValue.equals(expectedValue), "The actual value does not match the expected value");
}
}

private static long parse(DateTime time, String pattern)
{
return parseHiveTimestamp(DateTimeFormat.forPattern(pattern).print(time), nonDefaultTimeZone());
Expand Down