diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java index 85103c65307c..6ec48ebc6370 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java @@ -21,26 +21,46 @@ import java.nio.ByteBuffer; import java.util.Arrays; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.BytesWritable; +import org.apache.iceberg.util.ByteBuffers; -public final class IcebergBinaryObjectInspector extends IcebergPrimitiveObjectInspector - implements BinaryObjectInspector { +public abstract class IcebergBinaryObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements BinaryObjectInspector { - private static final IcebergBinaryObjectInspector INSTANCE = new IcebergBinaryObjectInspector(); + private static final IcebergBinaryObjectInspector BYTE_ARRAY = new IcebergBinaryObjectInspector() { + @Override + byte[] toByteArray(Object o) { + return (byte[]) o; + } + }; + + private static final IcebergBinaryObjectInspector BYTE_BUFFER = new IcebergBinaryObjectInspector() { + @Override + byte[] toByteArray(Object o) { + return ByteBuffers.toByteArray((ByteBuffer) o); + } + }; - public static IcebergBinaryObjectInspector get() { - return INSTANCE; + public static IcebergBinaryObjectInspector byteArray() { + return BYTE_ARRAY; + } + + public static IcebergBinaryObjectInspector byteBuffer() { + return BYTE_BUFFER; } private IcebergBinaryObjectInspector() { super(TypeInfoFactory.binaryTypeInfo); } + abstract byte[] toByteArray(Object object); + @Override public byte[] getPrimitiveJavaObject(Object o) { - return o == null ? null : ((ByteBuffer) o).array(); + return toByteArray(o); } @Override diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java index 2991540437c7..2af9a172e963 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java @@ -22,10 +22,13 @@ import java.sql.Date; import java.time.LocalDate; import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.util.DateTimeUtil; -public final class IcebergDateObjectInspector extends IcebergPrimitiveObjectInspector implements DateObjectInspector { +public final class IcebergDateObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements DateObjectInspector { private static final IcebergDateObjectInspector INSTANCE = new IcebergDateObjectInspector(); @@ -44,8 +47,7 @@ public Date getPrimitiveJavaObject(Object o) { @Override public DateWritable getPrimitiveWritableObject(Object o) { - Date date = getPrimitiveJavaObject(o); - return date == null ? null : new DateWritable(date); + return o == null ? null : new DateWritable(DateTimeUtil.daysFromDate((LocalDate) o)); } @Override diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java index 5d31ce814509..fa8f91c2e99c 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java @@ -25,11 +25,12 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -public final class IcebergDecimalObjectInspector extends IcebergPrimitiveObjectInspector +public final class IcebergDecimalObjectInspector extends AbstractPrimitiveJavaObjectInspector implements HiveDecimalObjectInspector { private static final Cache CACHE = Caffeine.newBuilder() @@ -49,16 +50,6 @@ private IcebergDecimalObjectInspector(int precision, int scale) { super(new DecimalTypeInfo(precision, scale)); } - @Override - public int precision() { - return ((DecimalTypeInfo) getTypeInfo()).precision(); - } - - @Override - public int scale() { - return ((DecimalTypeInfo) getTypeInfo()).scale(); - } - @Override public HiveDecimal getPrimitiveJavaObject(Object o) { return o == null ? null : HiveDecimal.create((BigDecimal) o); diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java index ca4875649415..5e251f7092a9 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java @@ -67,7 +67,7 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) { switch (primitiveType.typeId()) { case BINARY: - return IcebergBinaryObjectInspector.get(); + return IcebergBinaryObjectInspector.byteBuffer(); case BOOLEAN: primitiveTypeInfo = TypeInfoFactory.booleanTypeInfo; break; @@ -79,6 +79,8 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) { case DOUBLE: primitiveTypeInfo = TypeInfoFactory.doubleTypeInfo; break; + case FIXED: + return IcebergBinaryObjectInspector.byteArray(); case FLOAT: primitiveTypeInfo = TypeInfoFactory.floatTypeInfo; break; @@ -89,15 +91,14 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) { primitiveTypeInfo = TypeInfoFactory.longTypeInfo; break; case STRING: + case UUID: primitiveTypeInfo = TypeInfoFactory.stringTypeInfo; break; case TIMESTAMP: boolean adjustToUTC = ((Types.TimestampType) primitiveType).shouldAdjustToUTC(); return IcebergTimestampObjectInspector.get(adjustToUTC); - case FIXED: case TIME: - case UUID: default: throw new IllegalArgumentException(primitiveType.typeId() + " type is not supported"); } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java deleted file mode 100644 index 53c3560c2dd7..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred.serde.objectinspector; - -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; - -abstract class IcebergPrimitiveObjectInspector implements PrimitiveObjectInspector { - - private final PrimitiveTypeInfo typeInfo; - - protected IcebergPrimitiveObjectInspector(PrimitiveTypeInfo typeInfo) { - this.typeInfo = typeInfo; - } - - @Override - public Category getCategory() { - return typeInfo.getCategory(); - } - - @Override - public String getTypeName() { - return typeInfo.getTypeName(); - } - - @Override - public PrimitiveTypeInfo getTypeInfo() { - return typeInfo; - } - - @Override - public PrimitiveObjectInspector.PrimitiveCategory getPrimitiveCategory() { - return typeInfo.getPrimitiveCategory(); - } - - @Override - public Class getJavaPrimitiveClass() { - return typeInfo.getPrimitiveJavaClass(); - } - - @Override - public Class getPrimitiveWritableClass() { - return typeInfo.getPrimitiveWritableClass(); - } - - @Override - public boolean preferWritable() { - return false; - } - - @Override - public int precision() { - return 0; - } - - @Override - public int scale() { - return 0; - } - -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java index 7005e4239708..c49396b1f168 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -157,12 +158,12 @@ public boolean equals(Object o) { } IcebergRecordStructField that = (IcebergRecordStructField) o; - return field.equals(that.field) && oi.equals(that.oi); + return field.equals(that.field) && oi.equals(that.oi) && position == that.position; } @Override public int hashCode() { - return 31 * field.hashCode() + oi.hashCode(); + return Objects.hash(field, oi, position); } } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java index 569267df8496..974e59f69194 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java @@ -22,34 +22,42 @@ import java.sql.Timestamp; import java.time.LocalDateTime; import java.time.OffsetDateTime; -import java.util.function.Function; import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -public final class IcebergTimestampObjectInspector extends IcebergPrimitiveObjectInspector - implements TimestampObjectInspector { +public abstract class IcebergTimestampObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements TimestampObjectInspector { - private static final IcebergTimestampObjectInspector INSTANCE_WITH_ZONE = - new IcebergTimestampObjectInspector(o -> ((OffsetDateTime) o).toLocalDateTime()); + private static final IcebergTimestampObjectInspector INSTANCE_WITH_ZONE = new IcebergTimestampObjectInspector() { + @Override + LocalDateTime toLocalDateTime(Object o) { + return ((OffsetDateTime) o).toLocalDateTime(); + } + }; - private static final IcebergTimestampObjectInspector INSTANCE_WITHOUT_ZONE = - new IcebergTimestampObjectInspector(o -> (LocalDateTime) o); + private static final IcebergTimestampObjectInspector INSTANCE_WITHOUT_ZONE = new IcebergTimestampObjectInspector() { + @Override + LocalDateTime toLocalDateTime(Object o) { + return (LocalDateTime) o; + } + }; public static IcebergTimestampObjectInspector get(boolean adjustToUTC) { return adjustToUTC ? INSTANCE_WITH_ZONE : INSTANCE_WITHOUT_ZONE; } - private final Function cast; - - private IcebergTimestampObjectInspector(Function cast) { + private IcebergTimestampObjectInspector() { super(TypeInfoFactory.timestampTypeInfo); - this.cast = cast; } + + abstract LocalDateTime toLocalDateTime(Object object); + @Override public Timestamp getPrimitiveJavaObject(Object o) { - return o == null ? null : Timestamp.valueOf(cast.apply(o)); + return o == null ? null : Timestamp.valueOf(toLocalDateTime(o)); } @Override diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java index 5d88da53cd6c..f28e78b4867b 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java @@ -31,8 +31,8 @@ public class TestIcebergBinaryObjectInspector { @Test - public void testIcebergBinaryObjectInspector() { - BinaryObjectInspector oi = IcebergBinaryObjectInspector.get(); + public void testIcebergByteArrayObjectInspector() { + BinaryObjectInspector oi = IcebergBinaryObjectInspector.byteArray(); Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory()); @@ -48,11 +48,49 @@ public void testIcebergBinaryObjectInspector() { Assert.assertNull(oi.getPrimitiveWritableObject(null)); byte[] bytes = new byte[] {0, 1}; - ByteBuffer buffer = ByteBuffer.wrap(bytes); + Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(bytes)); + Assert.assertEquals(new BytesWritable(bytes), oi.getPrimitiveWritableObject(bytes)); + + byte[] copy = (byte[]) oi.copyObject(bytes); + + Assert.assertArrayEquals(bytes, copy); + Assert.assertNotSame(bytes, copy); + + Assert.assertFalse(oi.preferWritable()); + } + + @Test + public void testIcebergByteBufferObjectInspector() { + BinaryObjectInspector oi = IcebergBinaryObjectInspector.byteBuffer(); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(byte[].class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(BytesWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + byte[] bytes = new byte[] {0, 1, 2, 3}; + + ByteBuffer buffer = ByteBuffer.wrap(bytes); Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(buffer)); Assert.assertEquals(new BytesWritable(bytes), oi.getPrimitiveWritableObject(buffer)); + ByteBuffer slice = ByteBuffer.wrap(bytes, 1, 2).slice(); + Assert.assertArrayEquals(new byte[] {1, 2}, oi.getPrimitiveJavaObject(slice)); + Assert.assertEquals(new BytesWritable(new byte[] {1, 2}), oi.getPrimitiveWritableObject(slice)); + + slice.position(1); + Assert.assertArrayEquals(new byte[] {2}, oi.getPrimitiveJavaObject(slice)); + Assert.assertEquals(new BytesWritable(new byte[] {2}), oi.getPrimitiveWritableObject(slice)); + byte[] copy = (byte[]) oi.copyObject(bytes); Assert.assertArrayEquals(bytes, copy); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java index b280b05fb86d..908c9dbfbef8 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java @@ -38,31 +38,37 @@ public class TestIcebergObjectInspector { + private int id = 0; + private final Schema schema = new Schema( - required(0, "binary_field", Types.BinaryType.get(), "binary comment"), - required(1, "boolean_field", Types.BooleanType.get(), "boolean comment"), - required(2, "date_field", Types.DateType.get(), "date comment"), - required(3, "decimal_field", Types.DecimalType.of(38, 18), "decimal comment"), - required(4, "double_field", Types.DoubleType.get(), "double comment"), - required(5, "float_field", Types.FloatType.get(), "float comment"), - required(6, "integer_field", Types.IntegerType.get(), "integer comment"), - required(7, "long_field", Types.LongType.get(), "long comment"), - required(8, "string_field", Types.StringType.get(), "string comment"), - required(9, "timestamp_field", Types.TimestampType.withoutZone(), "timestamp comment"), - required(10, "timestamptz_field", Types.TimestampType.withZone(), "timestamptz comment"), - required(11, "list_field", - Types.ListType.ofRequired(12, Types.StringType.get()), "list comment"), - required(13, "map_field", - Types.MapType.ofRequired(14, 15, Types.StringType.get(), Types.IntegerType.get()), + required(id++, "binary_field", Types.BinaryType.get(), "binary comment"), + required(id++, "boolean_field", Types.BooleanType.get(), "boolean comment"), + required(id++, "date_field", Types.DateType.get(), "date comment"), + required(id++, "decimal_field", Types.DecimalType.of(38, 18), "decimal comment"), + required(id++, "double_field", Types.DoubleType.get(), "double comment"), + required(id++, "fixed_field", Types.FixedType.ofLength(3), "fixed comment"), + required(id++, "float_field", Types.FloatType.get(), "float comment"), + required(id++, "integer_field", Types.IntegerType.get(), "integer comment"), + required(id++, "long_field", Types.LongType.get(), "long comment"), + required(id++, "string_field", Types.StringType.get(), "string comment"), + required(id++, "timestamp_field", Types.TimestampType.withoutZone(), "timestamp comment"), + required(id++, "timestamptz_field", Types.TimestampType.withZone(), "timestamptz comment"), + required(id++, "uuid_field", Types.UUIDType.get(), "uuid comment"), + required(id++, "list_field", + Types.ListType.ofRequired(id++, Types.StringType.get()), "list comment"), + required(id++, "map_field", + Types.MapType.ofRequired(id++, id++, Types.StringType.get(), Types.IntegerType.get()), "map comment"), - required(16, "struct_field", Types.StructType.of( - Types.NestedField.required(17, "nested_field", Types.StringType.get(), "nested field comment")), + required(id++, "struct_field", Types.StructType.of( + Types.NestedField.required(id++, "nested_field", Types.StringType.get(), "nested field comment")), "struct comment" ) ); @Test public void testIcebergObjectInspector() { + int fieldId = 0; + ObjectInspector oi = IcebergObjectInspector.create(schema); Assert.assertNotNull(oi); Assert.assertEquals(ObjectInspector.Category.STRUCT, oi.getCategory()); @@ -71,119 +77,131 @@ public void testIcebergObjectInspector() { // binary StructField binaryField = soi.getStructFieldRef("binary_field"); - Assert.assertEquals(0, binaryField.getFieldID()); + Assert.assertEquals(fieldId++, binaryField.getFieldID()); Assert.assertEquals("binary_field", binaryField.getFieldName()); Assert.assertEquals("binary comment", binaryField.getFieldComment()); - Assert.assertEquals(IcebergBinaryObjectInspector.get(), binaryField.getFieldObjectInspector()); + Assert.assertEquals(IcebergBinaryObjectInspector.byteBuffer(), binaryField.getFieldObjectInspector()); // boolean StructField booleanField = soi.getStructFieldRef("boolean_field"); - Assert.assertEquals(1, booleanField.getFieldID()); + Assert.assertEquals(fieldId++, booleanField.getFieldID()); Assert.assertEquals("boolean_field", booleanField.getFieldName()); Assert.assertEquals("boolean comment", booleanField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(boolean.class), booleanField.getFieldObjectInspector()); // date StructField dateField = soi.getStructFieldRef("date_field"); - Assert.assertEquals(2, dateField.getFieldID()); + Assert.assertEquals(fieldId++, dateField.getFieldID()); Assert.assertEquals("date_field", dateField.getFieldName()); Assert.assertEquals("date comment", dateField.getFieldComment()); Assert.assertEquals(IcebergDateObjectInspector.get(), dateField.getFieldObjectInspector()); // decimal StructField decimalField = soi.getStructFieldRef("decimal_field"); - Assert.assertEquals(3, decimalField.getFieldID()); + Assert.assertEquals(fieldId++, decimalField.getFieldID()); Assert.assertEquals("decimal_field", decimalField.getFieldName()); Assert.assertEquals("decimal comment", decimalField.getFieldComment()); Assert.assertEquals(IcebergDecimalObjectInspector.get(38, 18), decimalField.getFieldObjectInspector()); // double StructField doubleField = soi.getStructFieldRef("double_field"); - Assert.assertEquals(4, doubleField.getFieldID()); + Assert.assertEquals(fieldId++, doubleField.getFieldID()); Assert.assertEquals("double_field", doubleField.getFieldName()); Assert.assertEquals("double comment", doubleField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(double.class), doubleField.getFieldObjectInspector()); + // fixed + StructField fixedField = soi.getStructFieldRef("fixed_field"); + Assert.assertEquals(fieldId++, fixedField.getFieldID()); + Assert.assertEquals("fixed_field", fixedField.getFieldName()); + Assert.assertEquals("fixed comment", fixedField.getFieldComment()); + Assert.assertEquals(IcebergBinaryObjectInspector.byteArray(), fixedField.getFieldObjectInspector()); + // float StructField floatField = soi.getStructFieldRef("float_field"); - Assert.assertEquals(5, floatField.getFieldID()); + Assert.assertEquals(fieldId++, floatField.getFieldID()); Assert.assertEquals("float_field", floatField.getFieldName()); Assert.assertEquals("float comment", floatField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(float.class), floatField.getFieldObjectInspector()); // integer StructField integerField = soi.getStructFieldRef("integer_field"); - Assert.assertEquals(6, integerField.getFieldID()); + Assert.assertEquals(fieldId++, integerField.getFieldID()); Assert.assertEquals("integer_field", integerField.getFieldName()); Assert.assertEquals("integer comment", integerField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(int.class), integerField.getFieldObjectInspector()); // long StructField longField = soi.getStructFieldRef("long_field"); - Assert.assertEquals(7, longField.getFieldID()); + Assert.assertEquals(fieldId++, longField.getFieldID()); Assert.assertEquals("long_field", longField.getFieldName()); Assert.assertEquals("long comment", longField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(long.class), longField.getFieldObjectInspector()); // string StructField stringField = soi.getStructFieldRef("string_field"); - Assert.assertEquals(8, stringField.getFieldID()); + Assert.assertEquals(fieldId++, stringField.getFieldID()); Assert.assertEquals("string_field", stringField.getFieldName()); Assert.assertEquals("string comment", stringField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(String.class), stringField.getFieldObjectInspector()); // timestamp without tz StructField timestampField = soi.getStructFieldRef("timestamp_field"); - Assert.assertEquals(9, timestampField.getFieldID()); + Assert.assertEquals(fieldId++, timestampField.getFieldID()); Assert.assertEquals("timestamp_field", timestampField.getFieldName()); Assert.assertEquals("timestamp comment", timestampField.getFieldComment()); Assert.assertEquals(IcebergTimestampObjectInspector.get(false), timestampField.getFieldObjectInspector()); // timestamp with tz StructField timestampTzField = soi.getStructFieldRef("timestamptz_field"); - Assert.assertEquals(10, timestampTzField.getFieldID()); + Assert.assertEquals(fieldId++, timestampTzField.getFieldID()); Assert.assertEquals("timestamptz_field", timestampTzField.getFieldName()); Assert.assertEquals("timestamptz comment", timestampTzField.getFieldComment()); Assert.assertEquals(IcebergTimestampObjectInspector.get(true), timestampTzField.getFieldObjectInspector()); + // UUID + StructField uuidField = soi.getStructFieldRef("uuid_field"); + Assert.assertEquals(fieldId++, uuidField.getFieldID()); + Assert.assertEquals("uuid_field", uuidField.getFieldName()); + Assert.assertEquals("uuid comment", uuidField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(String.class), uuidField.getFieldObjectInspector()); + // list StructField listField = soi.getStructFieldRef("list_field"); - Assert.assertEquals(11, listField.getFieldID()); + Assert.assertEquals(fieldId++, listField.getFieldID()); Assert.assertEquals("list_field", listField.getFieldName()); Assert.assertEquals("list comment", listField.getFieldComment()); Assert.assertEquals(getListObjectInspector(String.class), listField.getFieldObjectInspector()); + // skip element id + fieldId++; + // map StructField mapField = soi.getStructFieldRef("map_field"); - Assert.assertEquals(13, mapField.getFieldID()); + Assert.assertEquals(fieldId++, mapField.getFieldID()); Assert.assertEquals("map_field", mapField.getFieldName()); Assert.assertEquals("map comment", mapField.getFieldComment()); Assert.assertEquals(getMapObjectInspector(String.class, int.class), mapField.getFieldObjectInspector()); + // skip key and value ids + fieldId += 2; + // struct StructField structField = soi.getStructFieldRef("struct_field"); - Assert.assertEquals(16, structField.getFieldID()); + Assert.assertEquals(fieldId, structField.getFieldID()); Assert.assertEquals("struct_field", structField.getFieldName()); Assert.assertEquals("struct comment", structField.getFieldComment()); ObjectInspector expectedObjectInspector = new IcebergRecordObjectInspector( - (Types.StructType) schema.findType(16), ImmutableList.of(getPrimitiveObjectInspector(String.class))); + (Types.StructType) schema.findType(fieldId), ImmutableList.of(getPrimitiveObjectInspector(String.class))); Assert.assertEquals(expectedObjectInspector, structField.getFieldObjectInspector()); } @Test public void testIcebergObjectInspectorUnsupportedTypes() { - AssertHelpers.assertThrows( - "Hive does not support fixed type", IllegalArgumentException.class, "FIXED type is not supported", - () -> IcebergObjectInspector.create(required(1, "fixed_field", Types.FixedType.ofLength(1)))); - AssertHelpers.assertThrows( "Hive does not support time type", IllegalArgumentException.class, "TIME type is not supported", () -> IcebergObjectInspector.create(required(1, "time_field", Types.TimeType.get()))); - - AssertHelpers.assertThrows( - "Hive does not support UUID type", IllegalArgumentException.class, "UUID type is not supported", - () -> IcebergObjectInspector.create(required(1, "uuid_field", Types.UUIDType.get()))); } private static ObjectInspector getPrimitiveObjectInspector(Class clazz) {