Skip to content

Ensure nullable partitioning column is handled properly #15470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
highker merged 1 commit into from
Dec 8, 2020
Merged

Ensure nullable partitioning column is handled properly #15470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,8 +524,7 @@ private static boolean isValidPartitionType(Type type)
public static NullableValue parsePartitionValue(String partitionName, String value, Type type, DateTimeZone timeZone)
{
verifyPartitionTypeSupported(partitionName, type);

boolean isNull = HIVE_DEFAULT_DYNAMIC_PARTITION.equals(value);
boolean isNull = HIVE_DEFAULT_DYNAMIC_PARTITION.equals(value) || isHiveNull(value.getBytes(UTF_8));

if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
Expand Down
17 changes: 17 additions & 0 deletions ...est/java/com/facebook/presto/hive/TestHiveDistributedJoinQueriesWithDynamicFiltering.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,23 @@ public void testJoinDynamicFilteringMultiJoin()
assertQuery(session, query, "SELECT 1, 1, 1");
}

@Test
public void testJoinOnNullPartitioning()
{
assertUpdate("CREATE TABLE t3(c2 bigint, c1 bigint)");
assertUpdate("INSERT INTO t3 VALUES(null, 2)", 1);
assertUpdate("CREATE TABLE t4(c2 bigint, c1 bigint) with(partitioned_by=array['c1'])");
assertUpdate("INSERT INTO t4 VALUES(null, null), (2,2)", 2);

String query = "select * from t3, t4 where t3.c1=t4.c2";
Session session = Session.builder(getSession())
.setSystemProperty(ENABLE_DYNAMIC_FILTERING, "true")
.setSystemProperty(JOIN_DISTRIBUTION_TYPE, FeaturesConfig.JoinDistributionType.AUTOMATIC.name())
.setSystemProperty(JOIN_REORDERING_STRATEGY, FeaturesConfig.JoinReorderingStrategy.AUTOMATIC.name())
.build();
assertQuery(session, query, "SELECT null, 2, 2, 2");
}

private OperatorStats searchScanFilterAndProjectOperatorStats(QueryId queryId, String tableName)
{
DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
Expand Down
10 changes: 10 additions & 0 deletions ...rc/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
*/
package com.facebook.presto.hive.statistics;

import com.facebook.presto.common.predicate.NullableValue;
import com.facebook.presto.common.type.DecimalType;
import com.facebook.presto.common.type.Type;
import com.facebook.presto.hive.HiveBasicStatistics;
Expand Down Expand Up @@ -87,6 +88,7 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;

public class TestMetastoreHiveStatisticsProvider
{
Expand Down Expand Up @@ -116,6 +118,14 @@ public void testGetPartitionsSample()
assertEquals(getPartitionsSample(ImmutableList.of(p1, p2, p3, p4, p5), 3), ImmutableList.of(p1, p5, p4));
}

@Test
public void testNullablePartitionValue()
{
HivePartition part = partition("p1=name/p2=\\N");
NullableValue verify = new NullableValue(BIGINT, null);
assertTrue(part.getKeys().containsValue(verify));
}

@Test
public void testValidatePartitionStatistics()
{
Expand Down