Build: Enable revapi on core/parquet/orc/common/data modules & fix API breaks

.palantir/revapi.yml

@@ -1,4 +1,70 @@
 acceptedBreaks:
+  "1.0.0":
+    org.apache.iceberg:iceberg-core:
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.BaseMetadataTable"
+      new: "class org.apache.iceberg.BaseMetadataTable"
+      justification: "Serialization changed"
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.BaseTable"
+      new: "class org.apache.iceberg.BaseTable"
+      justification: "Serialization changed"
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.BaseTransaction.TransactionTable"
+      new: "class org.apache.iceberg.BaseTransaction.TransactionTable"
+      justification: "Serialization changed"
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.MetricsConfig"
+      new: "class org.apache.iceberg.MetricsConfig"
+      justification: "Serialization changed"
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.SerializableTable"
+      new: "class org.apache.iceberg.SerializableTable"
+      justification: "Serialization changed"
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.TableMetadata"
+      new: "class org.apache.iceberg.TableMetadata"
+      justification: "Serialization changed"
+    - code: "java.field.noLongerConstant"
+      old: "field org.apache.iceberg.TableProperties.DELETE_MODE_DEFAULT"
+      new: "field org.apache.iceberg.TableProperties.DELETE_MODE_DEFAULT"
+      justification: "Refactorings"
+    - code: "java.field.noLongerConstant"
+      old: "field org.apache.iceberg.TableProperties.MERGE_MODE_DEFAULT"
+      new: "field org.apache.iceberg.TableProperties.MERGE_MODE_DEFAULT"
+      justification: "Refactorings"
+    - code: "java.field.noLongerConstant"
+      old: "field org.apache.iceberg.TableProperties.UPDATE_MODE_DEFAULT"
+      new: "field org.apache.iceberg.TableProperties.UPDATE_MODE_DEFAULT"
+      justification: "Refactorings"
+    - code: "java.method.addedToInterface"
+      new: "method org.apache.hadoop.conf.Configuration org.apache.hadoop.conf.Configurable::getConf()\
+        \ @ org.apache.iceberg.hadoop.HadoopConfigurable"
+      justification: "New method was added to org.apache.hadoop.conf.Configurable"
+    - code: "java.method.addedToInterface"
+      new: "method void org.apache.hadoop.conf.Configurable::setConf(org.apache.hadoop.conf.Configuration)\
+        \ @ org.apache.iceberg.hadoop.HadoopConfigurable"
+      justification: "New method was added to org.apache.hadoop.conf.Configurable"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.TableMetadata.Builder org.apache.iceberg.TableMetadata.Builder::removeBranch(java.lang.String)"
+      justification: "Removing deprecated method"
+    - code: "java.method.removed"
+      old: "method void org.apache.iceberg.deletes.EqualityDeleteWriter<T>::delete(T)"
+      justification: "Removing deprecated method"
+    - code: "java.method.removed"
+      old: "method void org.apache.iceberg.deletes.EqualityDeleteWriter<T>::deleteAll(java.lang.Iterable<T>)"
+      justification: "Removing deprecated method"
+    - code: "java.method.removed"
+      old: "method void org.apache.iceberg.deletes.PositionDeleteWriter<T>::delete(java.lang.CharSequence,\
+        \ long)"
+      justification: "Removing deprecated method"
+    - code: "java.method.removed"
+      old: "method void org.apache.iceberg.deletes.PositionDeleteWriter<T>::delete(java.lang.CharSequence,\
+        \ long, T)"
+      justification: "Removing deprecated method"
+    - code: "java.method.removed"
+      old: "method void org.apache.iceberg.io.DataWriter<T>::add(T)"
+      justification: "Removing deprecated method"
   apache-iceberg-0.14.0:
     org.apache.iceberg:iceberg-api:
     - code: "java.class.defaultSerializationChanged"
@@ -11,15 +77,15 @@ acceptedBreaks:
     - code: "java.method.addedToInterface"
       new: "method java.lang.String org.apache.iceberg.expressions.Reference<T>::name()"
       justification: "All subclasses implement name"
-    - code: "java.method.addedToInterface"
-      new: "method org.apache.iceberg.actions.MigrateTable org.apache.iceberg.actions.MigrateTable::dropBackup()"
-      justification: "Adding new functionality to allow for dropping backup table"
     - code: "java.method.addedToInterface"
       new: "method java.util.List<org.apache.iceberg.StatisticsFile> org.apache.iceberg.Table::statisticsFiles()"
       justification: "new API method"
     - code: "java.method.removed"
       old: "method java.lang.Iterable<org.apache.iceberg.DataFile> org.apache.iceberg.Snapshot::addedFiles()"
       justification: "Deprecations for 1.0 release"
+    - code: "java.method.removed"
+      old: "method java.lang.Iterable<org.apache.iceberg.DataFile> org.apache.iceberg.Snapshot::deletedFiles()"
+      justification: "Deprecations for 1.0 release"
     - code: "java.method.removed"
       old: "method java.util.List<org.apache.iceberg.ManifestFile> org.apache.iceberg.Snapshot::allManifests()"
       justification: "Deprecations for 1.0 release"
@@ -29,9 +95,6 @@ acceptedBreaks:
     - code: "java.method.removed"
       old: "method java.util.List<org.apache.iceberg.ManifestFile> org.apache.iceberg.Snapshot::deleteManifests()"
       justification: "Deprecations for 1.0 release"
-    - code: "java.method.removed"
-      old: "method java.lang.Iterable<org.apache.iceberg.DataFile> org.apache.iceberg.Snapshot::deletedFiles()"
-      justification: "Deprecations for 1.0 release"
     - code: "java.method.removed"
       old: "method org.apache.iceberg.OverwriteFiles org.apache.iceberg.OverwriteFiles::validateNoConflictingAppends(org.apache.iceberg.expressions.Expression)"
       justification: "Deprecations for 1.0 release"
@@ -41,9 +104,6 @@ acceptedBreaks:
     - code: "java.method.removed"
       old: "method org.apache.iceberg.RowDelta org.apache.iceberg.RowDelta::validateNoConflictingAppends(org.apache.iceberg.expressions.Expression)"
       justification: "Deprecations for 1.0 release"
-    - code: "java.method.addedToInterface"
-      new: "method org.apache.iceberg.TableScan org.apache.iceberg.TableScan::useRef(java.lang.String)"
-      justification: "Adding table scan APIs to support scanning from refs"
   release-base-0.13.0:
     org.apache.iceberg:iceberg-api:
     - code: "java.class.defaultSerializationChanged"

api/src/main/java/org/apache/iceberg/TableScan.java

@@ -45,7 +45,9 @@ public interface TableScan extends Scan<TableScan, FileScanTask, CombinedScanTas
    * @return a new scan based on the given reference.
    * @throws IllegalArgumentException if a reference with the given name could not be found
    */
-  TableScan useRef(String ref);
+  default TableScan useRef(String ref) {
+    throw new UnsupportedOperationException("Using a reference is not supported");
+  }
 
   /**
    * Create a new {@link TableScan} from this scan's configuration that will use the most recent

api/src/main/java/org/apache/iceberg/actions/MigrateTable.java

@@ -46,7 +46,9 @@ public interface MigrateTable extends Action<MigrateTable, MigrateTable.Result>
    *
    * @return this for method chaining
    */
-  MigrateTable dropBackup();
+  default MigrateTable dropBackup() {
+    throw new UnsupportedOperationException("Dropping a backup is not supported");
+  }
 
   /** The action result that contains a summary of the execution. */
   interface Result {

build.gradle

@@ -91,6 +91,7 @@ dependencyRecommendations {
 }
 
 def projectVersion = getProjectVersion()
+final REVAPI_PROJECTS = ["iceberg-api", "iceberg-core", "iceberg-parquet", "iceberg-orc", "iceberg-common", "iceberg-data"]
 
 allprojects {
   group = "org.apache.iceberg"
@@ -105,6 +106,15 @@ subprojects {
   apply plugin: 'nebula.dependency-recommender'
   apply plugin: 'java-library'
 
+  if (project.name in REVAPI_PROJECTS) {
+    apply plugin: 'com.palantir.revapi'
+    revapi {
+      oldGroup = project.group
+      oldName = project.name
+      oldVersion = "1.0.0"
+    }
+  }
+
   configurations {
     testImplementation.extendsFrom compileOnly
 
@@ -225,14 +235,6 @@ project(':iceberg-bundled-guava') {
 }
 
 project(':iceberg-api') {
-  apply plugin: 'com.palantir.revapi'
-
-  revapi {
-    oldGroup = "org.apache.iceberg"
-    oldName = "iceberg-api"
-    oldVersion = "1.0.0"
-  }
-
   dependencies {
     implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
     compileOnly "com.google.errorprone:error_prone_annotations"

core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java

@@ -79,6 +79,20 @@ public ReplacePartitions validateNoConflictingData() {
     return this;
   }
 
+  /**
+   * Validate the current metadata.
+   *
+   * <p>Child operations can override this to add custom validation.
+   *
+   * @param currentMetadata current table metadata to validate
+   * @deprecated Will be removed in 1.2.0, use {@link SnapshotProducer#validate(TableMetadata,
+   *     Snapshot)}.
+   */
+  @Deprecated
+  public void validate(TableMetadata currentMetadata) {
+    super.validate(currentMetadata);
+  }
+
   @Override
   public void validate(TableMetadata currentMetadata, Snapshot snapshot) {
     if (validateConflictingData) {
@@ -100,6 +114,20 @@ public void validate(TableMetadata currentMetadata, Snapshot snapshot) {
     }
   }
 
+  /**
+   * Apply the update's changes to the base table metadata and return the new manifest list.
+   *
+   * @param base the base table metadata to apply changes to
+   * @return a manifest list for the new snapshot.
+   * @deprecated Will be removed in 1.2.0, use {@link BaseReplacePartitions#apply(TableMetadata,
+   *     Snapshot)}.
+   */
+  @Deprecated
+  @Override
+  public List<ManifestFile> apply(TableMetadata base) {
+    return super.apply(base);
+  }
+
   @Override
   public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
     if (dataSpec().fields().size() <= 0) {

core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java

@@ -173,6 +173,20 @@ private ManifestFile copyManifest(ManifestFile manifest) {
         summaryBuilder);
   }
 
+  /**
+   * Apply the update's changes to the base table metadata and return the new manifest list.
+   *
+   * @param base the base table metadata to apply changes to
+   * @return a manifest list for the new snapshot.
+   * @deprecated Will be removed in 1.2.0, use {@link BaseRewriteManifests#apply(TableMetadata,
+   *     Snapshot)}.
+   */
+  @Deprecated
+  @Override
+  public List<ManifestFile> apply(TableMetadata base) {
+    return super.apply(base);
+  }
+
   @Override
   public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
     List<ManifestFile> currentManifests = base.currentSnapshot().dataManifests(ops.io());

core/src/main/java/org/apache/iceberg/BaseTableScan.java

@@ -61,6 +61,16 @@ protected Long snapshotId() {
     return context().snapshotId();
   }
 
+  /**
+   * @return whether column stats are returned.
+   * @deprecated Will be removed in 1.2.0, use {@link TableScanContext#returnColumnStats()}
+   *     directly.
+   */
+  @Deprecated
+  protected boolean colStats() {
+    return context().returnColumnStats();
+  }
+
   protected Map<String, String> options() {
     return context().options();
   }

core/src/main/java/org/apache/iceberg/FastAppend.java

@@ -141,6 +141,19 @@ private ManifestFile copyManifest(ManifestFile manifest) {
         summaryBuilder);
   }
 
+  /**
+   * Apply the update's changes to the base table metadata and return the new manifest list.
+   *
+   * @param base the base table metadata to apply changes to
+   * @return a manifest list for the new snapshot.
+   * @deprecated Will be removed in 1.2.0, use {@link FastAppend#apply(TableMetadata, Snapshot)}.
+   */
+  @Deprecated
+  @Override
+  public List<ManifestFile> apply(TableMetadata base) {
+    return super.apply(base);
+  }
+
   @Override
   public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
     List<ManifestFile> newManifests = Lists.newArrayList();

core/src/main/java/org/apache/iceberg/ManifestReader.java

@@ -98,6 +98,38 @@ private String fileClass() {
   private Evaluator lazyEvaluator = null;
   private InclusiveMetricsEvaluator lazyMetricsEvaluator = null;
 
+  /**
+   * @deprecated Will be removed in 1.2.0, use {@link ManifestReader#ManifestReader(InputFile, int,
+   *     Map, InheritableMetadata, FileType)}.
+   */
+  @Deprecated
+  protected ManifestReader(
+      InputFile file,
+      Map<Integer, PartitionSpec> specsById,
+      InheritableMetadata inheritableMetadata,
+      FileType content) {
+    this.file = file;
+    this.inheritableMetadata = inheritableMetadata;
+    this.content = content;
+
+    Map<String, String> metadata = readMetadata(file);
+    int specId = TableMetadata.INITIAL_SPEC_ID;
+    String specProperty = metadata.get("partition-spec-id");
+    if (specProperty != null) {
+      specId = Integer.parseInt(specProperty);
+    }
+
+    if (specsById != null) {
+      this.spec = specsById.get(specId);
+    } else {
+      Schema schema = SchemaParser.fromJson(metadata.get("schema"));
+      this.spec =
+          PartitionSpecParser.fromJsonFields(schema, specId, metadata.get("partition-spec"));
+    }
+
+    this.fileSchema = new Schema(DataFile.getType(spec.partitionType()).fields());
+  }
+
   protected ManifestReader(
       InputFile file,
       int specId,
@@ -118,6 +150,19 @@ protected ManifestReader(
   }
 
   private <T extends ContentFile<T>> PartitionSpec readPartitionSpec(InputFile inputFile) {
+    Map<String, String> metadata = readMetadata(inputFile);
+
+    int specId = TableMetadata.INITIAL_SPEC_ID;
+    String specProperty = metadata.get("partition-spec-id");
+    if (specProperty != null) {
+      specId = Integer.parseInt(specProperty);
+    }
+
+    Schema schema = SchemaParser.fromJson(metadata.get("schema"));
+    return PartitionSpecParser.fromJsonFields(schema, specId, metadata.get("partition-spec"));
+  }
+
+  private static <T extends ContentFile<T>> Map<String, String> readMetadata(InputFile inputFile) {
     Map<String, String> metadata;
     try {
       try (AvroIterable<ManifestEntry<T>> headerReader =
@@ -130,15 +175,7 @@ private <T extends ContentFile<T>> PartitionSpec readPartitionSpec(InputFile inp
     } catch (IOException e) {
       throw new RuntimeIOException(e);
     }
-
-    int specId = TableMetadata.INITIAL_SPEC_ID;
-    String specProperty = metadata.get("partition-spec-id");
-    if (specProperty != null) {
-      specId = Integer.parseInt(specProperty);
-    }
-
-    Schema schema = SchemaParser.fromJson(metadata.get("schema"));
-    return PartitionSpecParser.fromJsonFields(schema, specId, metadata.get("partition-spec"));
+    return metadata;
   }
 
   public boolean isDeleteManifestReader() {

core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java

@@ -758,6 +758,20 @@ protected Map<String, String> summary() {
     return summaryBuilder.build();
   }
 
+  /**
+   * Apply the update's changes to the base table metadata and return the new manifest list.
+   *
+   * @param base the base table metadata to apply changes to
+   * @return a manifest list for the new snapshot.
+   * @deprecated Will be removed in 1.2.0, use {@link MergingSnapshotProducer#apply(TableMetadata,
+   *     Snapshot)}.
+   */
+  @Deprecated
+  @Override
+  public List<ManifestFile> apply(TableMetadata base) {
+    return super.apply(base);
+  }
+
   @Override
   public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
     // filter any existing manifests