[#2039] Support Default Value Semantis: use default values reading AVRO files

api/src/main/java/org/apache/iceberg/types/PruneColumns.java

@@ -52,11 +52,11 @@ public Type struct(Types.StructType struct, List<Type> fieldResults) {
       } else if (projectedType != null) {
         sameTypes = false; // signal that some types were altered
         if (field.isOptional()) {
-          selectedFields.add(
-              Types.NestedField.optional(field.fieldId(), field.name(), projectedType, field.doc()));
+          selectedFields.add(Types.NestedField.optional(
+              field.fieldId(), field.name(), projectedType, field.getDefaultValue(), field.doc()));
         } else {
-          selectedFields.add(
-              Types.NestedField.required(field.fieldId(), field.name(), projectedType, field.doc()));
+          selectedFields.add(Types.NestedField.required(
+              field.fieldId(), field.name(), projectedType, field.getDefaultValue(), field.doc()));
         }
       }
     }

api/src/main/java/org/apache/iceberg/types/Types.java

@@ -435,7 +435,6 @@ public static NestedField required(int id, String name, Type type, String doc) {
     }
 
     public static NestedField required(int id, String name, Type type, Object defaultValue, String doc) {
-      validateDefaultValueForRequiredField(defaultValue, name);
       return new NestedField(false, id, name, type, defaultValue, doc);
     }
 
@@ -451,28 +450,24 @@ public static NestedField of(int id, boolean isOptional, String name, Type type,
       return new NestedField(isOptional, id, name, type, defaultValue, doc);
     }
 
-    private static void validateDefaultValueForRequiredField(Object defaultValue, String fieldName) {
-      Preconditions.checkArgument(defaultValue != null,
-          "Cannot create NestedField with a null default for the required field: " +  fieldName);
-    }
-
     private static void validateDefaultValue(Object defaultValue, Type type) {
       if (defaultValue == null) {
         return;
       }
       switch (type.typeId()) {
         case STRUCT:
-          Preconditions.checkArgument(List.class.isInstance(defaultValue),
-              "defaultValue should be a List of Objects for StructType");
+          Preconditions.checkArgument(Map.class.isInstance(defaultValue),
+              "defaultValue should be a Map from fields names to values, for StructType");
           if (defaultValue == null) {
             return;
           }
-          List<Object> defaultList = (List) defaultValue;
-          Preconditions.checkArgument(defaultList.size() == type.asStructType().fields().size());
-          for (int i = 0; i < defaultList.size(); i++) {
-            NestedField.validateDefaultValue(defaultList.get(i), type.asStructType().fields().get(i).type);
+          Map<String, Object> defaultStruct = (Map<String, Object>) defaultValue;
+          Preconditions.checkArgument(defaultStruct.size() == type.asStructType().fields().size());
+          for (String fieldName : defaultStruct.keySet()) {
+            NestedField.validateDefaultValue(defaultStruct.get(fieldName), type.asStructType().field(fieldName).type);
           }
           break;
+
         case LIST:
           Preconditions.checkArgument(defaultValue instanceof ArrayList,
               "defaultValue should be an ArrayList of Objects, for ListType");
@@ -482,6 +477,7 @@ private static void validateDefaultValue(Object defaultValue, Type type) {
           }
           defaultArrayList.forEach(dv -> NestedField.validateDefaultValue(dv, type.asListType().elementField.type));
           break;
+
         case MAP:
           Preconditions.checkArgument(Map.class.isInstance(defaultValue),
               "defaultValue should be an instance of Map for MapType");
@@ -494,10 +490,18 @@ private static void validateDefaultValue(Object defaultValue, Type type) {
             NestedField.validateDefaultValue(e.getValue(), type.asMapType().valueField.type);
           }
           break;
+
+        case FIXED:
+        case BINARY:
+          Preconditions.checkArgument(byte[].class.isInstance(defaultValue),
+              "defaultValue should be an instance of byte[] for TypeId.%s, but defaultValue.class = %s",
+              type.typeId().name(), defaultValue.getClass().getCanonicalName());
+          break;
+
         default:
           Preconditions.checkArgument(type.typeId().javaClass().isInstance(defaultValue),
-              "defaultValue should be of same java class of the type, defaultValue class: " + defaultValue.getClass() +
-              ", type class: " + type.typeId().javaClass());
+              "defaultValue should be and instance of %s for TypeId.%s, but defaultValue.class = %s",
+              type.typeId().javaClass(), type.typeId().name(), defaultValue.getClass().getCanonicalName());
       }
     }
 

api/src/test/java/org/apache/iceberg/types/TestDefaultValuesForContainerTypes.java

@@ -21,7 +21,9 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -44,9 +46,9 @@ public static void beforeClass() {
 
   @Test
   public void testStructTypeDefault() {
-    List<Object> structDefaultvalue = new ArrayList<>();
-    structDefaultvalue.add(Integer.valueOf(1));
-    structDefaultvalue.add("two");
+    Map<String, Object> structDefaultvalue = new HashMap<>();
+    structDefaultvalue.put(intFieldType.name(), Integer.valueOf(1));
+    structDefaultvalue.put(stringFieldType.name(), "two");
     NestedField structField = NestedField.optional(2, "optionalStructField", structType, structDefaultvalue, "doc");
     Assert.assertTrue(structField.hasDefaultValue());
     Assert.assertEquals(structDefaultvalue, structField.getDefaultValue());

api/src/test/java/org/apache/iceberg/types/TestNestedFieldDefaultValues.java

@@ -50,6 +50,7 @@ public void testConstructorsValidCases() {
     // required constructors
     Assert.assertFalse(required(id, fieldName, fieldType).hasDefaultValue());
     Assert.assertFalse(required(id, fieldName, fieldType, doc).hasDefaultValue());
+    Assert.assertFalse(required(id, fieldName, fieldType, null, doc).hasDefaultValue());
     nestedFieldWithDefault = required(id, fieldName, fieldType, defaultValue, doc);
     Assert.assertTrue(nestedFieldWithDefault.hasDefaultValue());
     Assert.assertEquals(defaultValue, nestedFieldWithDefault.getDefaultValue());
@@ -65,22 +66,17 @@ public void testConstructorsValidCases() {
     Assert.assertEquals(defaultValue, nestedFieldWithDefault.getDefaultValue());
   }
 
-  @Test (expected =  IllegalArgumentException.class)
-  public void testRequiredNullDefault() {
-    // illegal case (required with null defaultValue)
-    required(id, fieldName, fieldType, null, doc);
-  }
-
   @Test (expected = IllegalArgumentException.class)
-  public void testRequiredWithDefaultNullDefault() {
-    // illegal case (required with null defaultValue)
-    required(id, fieldName, fieldType, null, null);
+  public void testOptionalWithInvalidDefaultValueClass() {
+    // class of default value does not match class of type
+    Long wrongClassDefaultValue = 100L;
+    optional(id, fieldName, fieldType, wrongClassDefaultValue, doc);
   }
 
   @Test (expected = IllegalArgumentException.class)
-  public void testOptionalWithInvalidDefaultValueClass() {
+  public void testReqiredWithInvalidDefaultValueClass() {
     // class of default value does not match class of type
     Long wrongClassDefaultValue = 100L;
-    optional(id, fieldName, fieldType, wrongClassDefaultValue, doc);
+    required(id, fieldName, fieldType, wrongClassDefaultValue, doc);
   }
 }