Skip to content

HIVE-27273: Iceberg: Upgrade iceberg to 1.2.1 #4252

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
deniskuzZ merged 2 commits into from
Apr 27, 2023
+142 −587
Merged

HIVE-27273: Iceberg: Upgrade iceberg to 1.2.1 #4252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a09cf19
Iceberg: Upgrade iceberg to 1.2.1
zhangbutao Apr 20, 2023
570db3c
Port apache/iceberg#7123
zhangbutao Apr 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ...rg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ private static Expression translate(ExpressionTree tree, List<PredicateLeaf> lea
case NOT:
return not(translate(childNodes.get(0), leaves));
case LEAF:
if (tree.getLeaf() >= leaves.size()) {
throw new UnsupportedOperationException("No more leaves are available");
}
return translateLeaf(leaves.get(tree.getLeaf()));
case CONSTANT:
throw new UnsupportedOperationException("CONSTANT operator is not supported");
Expand Down Expand Up @@ -107,6 +110,9 @@ private static Expression translateLeaf(PredicateLeaf leaf) {
return in(column, leafToLiteralList(leaf));
case BETWEEN:
List<Object> icebergLiterals = leafToLiteralList(leaf);
if (icebergLiterals.size() < 2) {
throw new UnsupportedOperationException("Missing leaf literals: " + leaf);
}
if (icebergLiterals.size() == 2) {
return and(greaterThanOrEqual(column, icebergLiterals.get(0)),
lessThanOrEqual(column, icebergLiterals.get(1)));
Expand Down
87 changes: 87 additions & 0 deletions ...ceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergFilterFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,14 @@
import java.sql.Timestamp;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hive.ql.io.sarg.ExpressionTree;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.expressions.And;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Literal;
Expand Down Expand Up @@ -134,6 +138,24 @@ public void testBetweenOperand() {
assertEquals(actual.right().op(), expected.right().op());
}

@Test
public void testUnsupportedBetweenOperandEmptyLeaves() {
SearchArgument.Builder builder = SearchArgumentFactory.newBuilder();
final SearchArgument arg =
new MockSearchArgument(
builder
.startAnd()
.between("salary", PredicateLeaf.Type.LONG, 9000L, 15000L)
.end()
.build());

AssertHelpers.assertThrows(
"must throw if leaves are empty in between operator",
UnsupportedOperationException.class,
"Missing leaf literals",
() -> HiveIcebergFilterFactory.generateFilterExpression(arg));
}

@Test
public void testIsNullOperand() {
SearchArgument.Builder builder = SearchArgumentFactory.newBuilder();
Expand Down Expand Up @@ -259,4 +281,69 @@ private void assertPredicatesMatch(UnboundPredicate expected, UnboundPredicate a
assertEquals(expected.literal(), actual.literal());
assertEquals(expected.ref().name(), actual.ref().name());
}

private static class MockSearchArgument implements SearchArgument {

private final SearchArgument delegate;

MockSearchArgument(SearchArgument original) {
delegate = original;
}

@Override
public ExpressionTree getExpression() {
return delegate.getExpression();
}

@Override
public ExpressionTree getCompactExpression() {
return null;
}

@Override
public TruthValue evaluate(TruthValue[] leaves) {
return delegate.evaluate(leaves);
}

@Override
public List<PredicateLeaf> getLeaves() {
return Collections.singletonList(
new PredicateLeaf() {
@Override
public Operator getOperator() {
return Operator.BETWEEN;
}

@Override
public Type getType() {
return Type.LONG;
}

@Override
public String getColumnName() {
return "salary";
}

@Override
public Object getLiteral() {
return null;
}

@Override
public List<Object> getLiteralList() {
return Collections.emptyList();
}

@Override
public int getId() {
return 0;
}

@Override
public String toString() {
return "Leaf[empty]";
}
});
}
}
}
18 changes: 18 additions & 0 deletions iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_metadata_tables.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe default.ice_meta_desc.entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -154,6 +155,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe default.ice_meta_desc.all_entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -186,6 +188,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe default.ice_meta_desc.delete_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand All @@ -207,6 +210,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe default.ice_meta_desc.metadata_log_entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -251,6 +255,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe default.ice_meta_desc.all_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand All @@ -272,6 +277,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand All @@ -294,6 +300,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -400,6 +407,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.all_entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -434,6 +442,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.delete_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand All @@ -456,6 +465,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.metadata_log_entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -503,6 +513,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.all_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand All @@ -525,6 +536,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe extended default.ice_meta_desc.files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand All @@ -546,6 +558,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe extended default.ice_meta_desc.entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -645,6 +658,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe extended default.ice_meta_desc.all_entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -678,6 +692,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.delete_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand All @@ -700,6 +715,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.metadata_log_entries
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down Expand Up @@ -747,6 +763,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: describe formatted default.ice_meta_desc.all_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
Expand All @@ -769,6 +786,7 @@ key_metadata binary Encryption key metadata blob
split_offsets array<bigint> Splittable offsets
equality_ids array<int> Equality comparison field IDs
sort_order_id int Sort order ID
readable_metrics struct<id:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:int,upper_bound:int>,value:struct<column_size:bigint,value_count:bigint,null_value_count:bigint,nan_value_count:bigint,lower_bound:string,upper_bound:string>> Column metrics in readable form
PREHOOK: query: drop table ice_meta_desc
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@ice_meta_desc
Expand Down
Loading