diff --git a/api/src/main/java/org/apache/iceberg/expressions/Literals.java b/api/src/main/java/org/apache/iceberg/expressions/Literals.java index 44c8b8be86d0..b4e31b78e06c 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/Literals.java +++ b/api/src/main/java/org/apache/iceberg/expressions/Literals.java @@ -39,6 +39,7 @@ import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ByteBuffers; import org.apache.iceberg.util.NaNUtil; class Literals { @@ -599,6 +600,11 @@ protected Type.TypeID typeId() { Object writeReplace() throws ObjectStreamException { return new SerializationProxies.FixedLiteralProxy(value()); } + + @Override + public String toString() { + return "0x" + ByteBuffers.encodeHexString(value()); + } } static class BinaryLiteral extends BaseLiteral { @@ -639,5 +645,10 @@ Object writeReplace() throws ObjectStreamException { protected Type.TypeID typeId() { return Type.TypeID.BINARY; } + + @Override + public String toString() { + return "0x" + ByteBuffers.encodeHexString(value()); + } } } diff --git a/api/src/main/java/org/apache/iceberg/util/ByteBuffers.java b/api/src/main/java/org/apache/iceberg/util/ByteBuffers.java index 213b222dc507..3aee403527d9 100644 --- a/api/src/main/java/org/apache/iceberg/util/ByteBuffers.java +++ b/api/src/main/java/org/apache/iceberg/util/ByteBuffers.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; +import org.apache.iceberg.relocated.com.google.common.io.BaseEncoding; public class ByteBuffers { @@ -58,6 +59,11 @@ public static ByteBuffer copy(ByteBuffer buffer) { return ByteBuffer.wrap(copyArray); } + public static String encodeHexString(ByteBuffer buffer) { + byte[] bytes = toByteArray(buffer); + return BaseEncoding.base16().encode(bytes); + } + private ByteBuffers() { } } diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java index b940fc137883..0af95e1a6f76 100644 --- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -19,7 +19,6 @@ package org.apache.iceberg.spark; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -636,17 +635,17 @@ public String predicate(UnboundPredicate pred) { case NOT_NAN: return "not_nan(" + pred.ref().name() + ")"; case LT: - return pred.ref().name() + " < " + sqlString(pred.literal()); + return pred.ref().name() + " < " + pred.literal(); case LT_EQ: - return pred.ref().name() + " <= " + sqlString(pred.literal()); + return pred.ref().name() + " <= " + pred.literal(); case GT: - return pred.ref().name() + " > " + sqlString(pred.literal()); + return pred.ref().name() + " > " + pred.literal(); case GT_EQ: - return pred.ref().name() + " >= " + sqlString(pred.literal()); + return pred.ref().name() + " >= " + pred.literal(); case EQ: - return pred.ref().name() + " = " + sqlString(pred.literal()); + return pred.ref().name() + " = " + pred.literal(); case NOT_EQ: - return pred.ref().name() + " != " + sqlString(pred.literal()); + return pred.ref().name() + " != " + pred.literal(); case STARTS_WITH: return pred.ref().name() + " LIKE '" + pred.literal() + "%'"; case IN: @@ -659,17 +658,7 @@ public String predicate(UnboundPredicate pred) { } private static String sqlString(List> literals) { - return literals.stream().map(DescribeExpressionVisitor::sqlString).collect(Collectors.joining(", ")); - } - - private static String sqlString(org.apache.iceberg.expressions.Literal lit) { - if (lit.value() instanceof String) { - return "'" + lit.value() + "'"; - } else if (lit.value() instanceof ByteBuffer) { - throw new IllegalArgumentException("Cannot convert bytes to SQL literal: " + lit); - } else { - return lit.value().toString(); - } + return literals.stream().map(Object::toString).collect(Collectors.joining(", ")); } } diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java index 93ceee3a27ee..f284b21b3d69 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java @@ -19,6 +19,7 @@ package org.apache.iceberg.spark; +import java.lang.reflect.Array; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -118,6 +119,8 @@ private Object[] toJava(Row row) { return row.getList(pos); } else if (value instanceof scala.collection.Map) { return row.getJavaMap(pos); + } else if (value.getClass().isArray() && value.getClass().getComponentType().isPrimitive()) { + return IntStream.range(0, Array.getLength(value)).mapToObj(i -> Array.get(value, i)).toArray(); } else { return value; } diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java index 846e234cba07..9342455d959f 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java @@ -37,6 +37,8 @@ public class TestSelect extends SparkCatalogTestBase { private int scanEventCount = 0; private ScanEvent lastScanEvent = null; + private final String binaryTableName = tableName("binary_table"); + public TestSelect(String catalogName, String implementation, Map config) { super(catalogName, implementation, config); @@ -52,6 +54,9 @@ public void createTables() { sql("CREATE TABLE %s (id bigint, data string, float float) USING iceberg", tableName); sql("INSERT INTO %s VALUES (1, 'a', 1.0), (2, 'b', 2.0), (3, 'c', float('NaN'))", tableName); + sql("CREATE TABLE %s (id bigint, binary binary) USING iceberg", binaryTableName); + sql("INSERT INTO %s VALUES (1, X''), (2, X'11'), (3, X'1111')", binaryTableName); + this.scanEventCount = 0; this.lastScanEvent = null; } @@ -59,6 +64,7 @@ public void createTables() { @After public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); + sql("DROP TABLE IF EXISTS %s", binaryTableName); } @Test @@ -120,4 +126,12 @@ public void testMetadataTables() { ImmutableList.of(row(ANY, ANY, null, "append", ANY, ANY)), sql("SELECT * FROM %s.snapshots", tableName)); } + + @Test + public void testFilterBinary() { + List expected = ImmutableList.of(row(3L, new Byte[]{0x11, 0x11})); + + assertEquals("Should return all expected rows", expected, + sql("SELECT * FROM %s where binary > X'1101'", binaryTableName)); + } }