Rebase LI-Iceberg changes on top of Apache Iceberg 0.14.1 release

.github/workflows/java-ci.yml

@@ -21,11 +21,11 @@ name: "Java CI"
 on:
   push:
     branches:
-    - 'master'
-    - '0.**'
-    tags:
-    - 'apache-iceberg-**'
+      - 'li-0.14.x'
+    tags-ignore: [v*] # release tags are autogenerated after a successful CI, no need to run CI against them
   pull_request:
+    branches:
+      - 'li-0.14.x'
     paths-ignore:
     - '.github/workflows/python-ci.yml'
     - '.github/workflows/spark-ci.yml'
@@ -53,28 +53,28 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        jvm: [8, 11]
+        jvm: [ 8, 11 ]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-java@v3
-      with:
-        distribution: zulu
-        java-version: ${{ matrix.jvm }}
-    - uses: actions/cache@v3
-      with:
-        path: ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
-        restore-keys: ${{ runner.os }}-gradle
-    - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -Pquick=true -x javadoc 
-    - uses: actions/upload-artifact@v3
-      if: failure()
-      with:
-        name: test logs
-        path: |
-          **/build/testlogs
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
+        with:
+          distribution: zulu
+          java-version: ${{ matrix.jvm }}
+      - uses: actions/cache@v3
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+      - run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -Pquick=true -x javadoc
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: test logs
+          path: |
+            **/build/testlogs
 
   build-checks:
     runs-on: ubuntu-20.04
@@ -84,7 +84,7 @@ jobs:
       with:
         distribution: zulu
         java-version: 8
-    - run: ./gradlew -DflinkVersions=1.13,1.14,1.15 -DsparkVersions=2.4,3.0,3.1,3.2,3.3 -DhiveVersions=2,3 build -x test -x javadoc -x integrationTest
+    - run: ./gradlew -DflinkVersions=1.13,1.14,1.15 -DsparkVersions=2.4,3.1 -DhiveVersions=2,3 build -x test -x javadoc -x integrationTest
 
   build-javadoc:
     runs-on: ubuntu-20.04
@@ -94,4 +94,32 @@ jobs:
       with:
         distribution: zulu
         java-version: 8
-    - run: ./gradlew -Pquick=true javadoc
+    - run: ./gradlew -P=true javadoc
+
+  release:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: '0' # https://github.com/shipkit/shipkit-changelog#fetch-depth-on-ci
+      - uses: actions/setup-java@v3
+        with:
+          distribution: zulu
+          java-version: 8
+      - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+      - run: ./gradlew build -DflinkVersions= -DsparkVersions=2.4,3.1 -DhiveVersions= -Pquick=true build -x javadoc
+      - name: Perform release
+        # Release job, only for pushes to the main development branch
+        if: ${{ github.event_name == 'push'
+          && github.ref == 'refs/heads/li-0.14.x'
+          && github.repository == 'linkedin/iceberg'
+          && !contains(toJSON(github.event.commits.*.message), '[skip release]') }}
+
+        run: ./gradlew -DflinkVersions= -DsparkVersions=2.4,3.1 -DhiveVersions= githubRelease publishToSonatype closeAndReleaseStagingRepository
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          SONATYPE_USER: ${{secrets.SONATYPE_USER}}
+          SONATYPE_PWD: ${{secrets.SONATYPE_PWD}}
+          PGP_KEY: ${{secrets.PGP_KEY}}
+          PGP_PWD: ${{secrets.PGP_PWD}}

.github/workflows/spark-ci.yml

@@ -21,8 +21,7 @@ name: "Spark CI"
 on:
   push:
     branches:
-    - 'master'
-    - '0.**'
+    - 'li-0.14.x'
     tags:
     - 'apache-iceberg-**'
   pull_request:
@@ -83,7 +82,7 @@ jobs:
     strategy:
       matrix:
         jvm: [8, 11]
-        spark: ['3.0', '3.1', '3.2', '3.3']
+        spark: ['3.1']
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -107,6 +106,7 @@ jobs:
             **/build/testlogs
 
   spark-3x-scala-2-13-tests:
+    if: ${{ false }}
     runs-on: ubuntu-20.04
     strategy:
       matrix:

api/src/main/java/org/apache/iceberg/types/PruneColumns.java

@@ -67,11 +67,11 @@ public Type struct(Types.StructType struct, List<Type> fieldResults) {
       } else if (projectedType != null) {
         sameTypes = false; // signal that some types were altered
         if (field.isOptional()) {
-          selectedFields.add(
-              Types.NestedField.optional(field.fieldId(), field.name(), projectedType, field.doc()));
+          selectedFields.add(Types.NestedField.optional(
+              field.fieldId(), field.name(), projectedType, field.getDefaultValue(), field.doc()));
         } else {
-          selectedFields.add(
-              Types.NestedField.required(field.fieldId(), field.name(), projectedType, field.doc()));
+          selectedFields.add(Types.NestedField.required(
+              field.fieldId(), field.name(), projectedType, field.getDefaultValue(), field.doc()));
         }
       }
     }

api/src/main/java/org/apache/iceberg/types/Types.java

@@ -415,42 +415,110 @@ public int hashCode() {
 
   public static class NestedField implements Serializable {
     public static NestedField optional(int id, String name, Type type) {
-      return new NestedField(true, id, name, type, null);
+      return new NestedField(true, id, name, type, null, null);
     }
 
     public static NestedField optional(int id, String name, Type type, String doc) {
-      return new NestedField(true, id, name, type, doc);
+      return new NestedField(true, id, name, type, null, doc);
+    }
+
+    public static NestedField optional(int id, String name, Type type, Object defaultValue, String doc) {
+      return new NestedField(true, id, name, type, defaultValue, doc);
     }
 
     public static NestedField required(int id, String name, Type type) {
-      return new NestedField(false, id, name, type, null);
+      return new NestedField(false, id, name, type, null, null);
     }
 
     public static NestedField required(int id, String name, Type type, String doc) {
-      return new NestedField(false, id, name, type, doc);
+      return new NestedField(false, id, name, type, null, doc);
+    }
+
+    public static NestedField required(int id, String name, Type type, Object defaultValue, String doc) {
+      return new NestedField(false, id, name, type, defaultValue, doc);
     }
 
     public static NestedField of(int id, boolean isOptional, String name, Type type) {
-      return new NestedField(isOptional, id, name, type, null);
+      return new NestedField(isOptional, id, name, type, null, null);
     }
 
     public static NestedField of(int id, boolean isOptional, String name, Type type, String doc) {
-      return new NestedField(isOptional, id, name, type, doc);
+      return new NestedField(isOptional, id, name, type, null, doc);
     }
 
+    public static NestedField of(int id, boolean isOptional, String name, Type type, Object defaultValue, String doc) {
+      return new NestedField(isOptional, id, name, type, defaultValue, doc);
+    }
+
+    private static void validateDefaultValue(Object defaultValue, Type type) {
+      if (defaultValue == null) {
+        return;
+      }
+      switch (type.typeId()) {
+        case STRUCT:
+          Preconditions.checkArgument(defaultValue instanceof Map,
+              "defaultValue should be a Map from fields names to values, for StructType");
+          Map<String, Object> defaultStruct = (Map<String, Object>) defaultValue;
+          if (defaultStruct.isEmpty()) {
+            return;
+          }
+          for (NestedField field : type.asStructType().fields()) {
+            validateDefaultValue(defaultStruct.getOrDefault(field.name(), field.getDefaultValue()), field.type());
+          }
+          break;
+
+        case LIST:
+          Preconditions.checkArgument(defaultValue instanceof List,
+              "defaultValue should be an List of Objects, for ListType");
+          List<Object> defaultList = (List<Object>) defaultValue;
+          if (defaultList.size() == 0) {
+            return;
+          }
+          defaultList.forEach(dv -> NestedField.validateDefaultValue(dv, type.asListType().elementField.type));
+          break;
+
+        case MAP:
+          Preconditions.checkArgument(defaultValue instanceof Map,
+              "defaultValue should be an instance of Map for MapType");
+          Map<Object, Object> defaultMap = (Map<Object, Object>) defaultValue;
+          if (defaultMap.isEmpty()) {
+            return;
+          }
+          for (Map.Entry<Object, Object> e : defaultMap.entrySet()) {
+            NestedField.validateDefaultValue(e.getKey(), type.asMapType().keyField.type);
+            NestedField.validateDefaultValue(e.getValue(), type.asMapType().valueField.type);
+          }
+          break;
+
+        case FIXED:
+        case BINARY:
+          Preconditions.checkArgument(defaultValue instanceof byte[],
+              "defaultValue should be an instance of byte[] for TypeId.%s, but defaultValue.class = %s",
+              type.typeId().name(), defaultValue.getClass().getCanonicalName());
+          break;
+
+        default:
+          Preconditions.checkArgument(type.typeId().javaClass().isInstance(defaultValue),
+              "defaultValue should be and instance of %s for TypeId.%s, but defaultValue.class = %s",
+              type.typeId().javaClass(), type.typeId().name(), defaultValue.getClass().getCanonicalName());
+      }
+    }
     private final boolean isOptional;
     private final int id;
     private final String name;
     private final Type type;
+    private final Object defaultValue;
     private final String doc;
 
-    private NestedField(boolean isOptional, int id, String name, Type type, String doc) {
+    private NestedField(boolean isOptional, int id, String name, Type type, Object defaultValue, String doc) {
       Preconditions.checkNotNull(name, "Name cannot be null");
       Preconditions.checkNotNull(type, "Type cannot be null");
+      validateDefaultValue(defaultValue, type);
       this.isOptional = isOptional;
       this.id = id;
       this.name = name;
       this.type = type;
+      this.defaultValue = defaultValue;
       this.doc = doc;
     }
 
@@ -462,7 +530,7 @@ public NestedField asOptional() {
       if (isOptional) {
         return this;
       }
-      return new NestedField(true, id, name, type, doc);
+      return new NestedField(true, id, name, type, defaultValue, doc);
     }
 
     public boolean isRequired() {
@@ -473,7 +541,15 @@ public NestedField asRequired() {
       if (!isOptional) {
         return this;
       }
-      return new NestedField(false, id, name, type, doc);
+      return new NestedField(false, id, name, type, defaultValue, doc);
+    }
+
+    public boolean hasDefaultValue() {
+      return defaultValue != null;
+    }
+
+    public Object getDefaultValue() {
+      return defaultValue;
     }
 
     public int fieldId() {
@@ -496,6 +572,7 @@ public String doc() {
     public String toString() {
       return String.format("%d: %s: %s %s",
           id, name, isOptional ? "optional" : "required", type) +
+          (hasDefaultValue() ? ", default value: " + defaultValue + ", " : "") +
           (doc != null ? " (" + doc + ")" : "");
     }
 
@@ -514,6 +591,9 @@ public boolean equals(Object o) {
         return false;
       } else if (!name.equals(that.name)) {
         return false;
+      } else if (!Objects.equals(defaultValue, that.defaultValue) &&
+          !Arrays.equals((byte[]) defaultValue, (byte[]) that.defaultValue)) {
+        return false;
       } else if (!Objects.equals(doc, that.doc)) {
         return false;
       }
@@ -522,7 +602,8 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-      return Objects.hash(NestedField.class, id, isOptional, name, type);
+      return hasDefaultValue() ? Objects.hash(NestedField.class, id, isOptional, name, type, defaultValue) :
+          Objects.hash(NestedField.class, id, isOptional, name, type);
     }
   }
 
@@ -740,7 +821,6 @@ public boolean equals(Object o) {
       } else if (!(o instanceof ListType)) {
         return false;
       }
-
       ListType listType = (ListType) o;
       return elementField.equals(listType.elementField);
     }