diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 615387108503..a4a7559b8e71 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -1,4 +1,70 @@ acceptedBreaks: + "1.0.0": + org.apache.iceberg:iceberg-core: + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.BaseMetadataTable" + new: "class org.apache.iceberg.BaseMetadataTable" + justification: "Serialization changed" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.BaseTable" + new: "class org.apache.iceberg.BaseTable" + justification: "Serialization changed" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.BaseTransaction.TransactionTable" + new: "class org.apache.iceberg.BaseTransaction.TransactionTable" + justification: "Serialization changed" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.MetricsConfig" + new: "class org.apache.iceberg.MetricsConfig" + justification: "Serialization changed" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.SerializableTable" + new: "class org.apache.iceberg.SerializableTable" + justification: "Serialization changed" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.TableMetadata" + new: "class org.apache.iceberg.TableMetadata" + justification: "Serialization changed" + - code: "java.field.noLongerConstant" + old: "field org.apache.iceberg.TableProperties.DELETE_MODE_DEFAULT" + new: "field org.apache.iceberg.TableProperties.DELETE_MODE_DEFAULT" + justification: "Refactorings" + - code: "java.field.noLongerConstant" + old: "field org.apache.iceberg.TableProperties.MERGE_MODE_DEFAULT" + new: "field org.apache.iceberg.TableProperties.MERGE_MODE_DEFAULT" + justification: "Refactorings" + - code: "java.field.noLongerConstant" + old: "field org.apache.iceberg.TableProperties.UPDATE_MODE_DEFAULT" + new: "field org.apache.iceberg.TableProperties.UPDATE_MODE_DEFAULT" + justification: "Refactorings" + - code: "java.method.addedToInterface" + new: "method org.apache.hadoop.conf.Configuration org.apache.hadoop.conf.Configurable::getConf()\ + \ @ org.apache.iceberg.hadoop.HadoopConfigurable" + justification: "New method was added to org.apache.hadoop.conf.Configurable" + - code: "java.method.addedToInterface" + new: "method void org.apache.hadoop.conf.Configurable::setConf(org.apache.hadoop.conf.Configuration)\ + \ @ org.apache.iceberg.hadoop.HadoopConfigurable" + justification: "New method was added to org.apache.hadoop.conf.Configurable" + - code: "java.method.removed" + old: "method org.apache.iceberg.TableMetadata.Builder org.apache.iceberg.TableMetadata.Builder::removeBranch(java.lang.String)" + justification: "Removing deprecated method" + - code: "java.method.removed" + old: "method void org.apache.iceberg.deletes.EqualityDeleteWriter::delete(T)" + justification: "Removing deprecated method" + - code: "java.method.removed" + old: "method void org.apache.iceberg.deletes.EqualityDeleteWriter::deleteAll(java.lang.Iterable)" + justification: "Removing deprecated method" + - code: "java.method.removed" + old: "method void org.apache.iceberg.deletes.PositionDeleteWriter::delete(java.lang.CharSequence,\ + \ long)" + justification: "Removing deprecated method" + - code: "java.method.removed" + old: "method void org.apache.iceberg.deletes.PositionDeleteWriter::delete(java.lang.CharSequence,\ + \ long, T)" + justification: "Removing deprecated method" + - code: "java.method.removed" + old: "method void org.apache.iceberg.io.DataWriter::add(T)" + justification: "Removing deprecated method" apache-iceberg-0.14.0: org.apache.iceberg:iceberg-api: - code: "java.class.defaultSerializationChanged" @@ -11,15 +77,15 @@ acceptedBreaks: - code: "java.method.addedToInterface" new: "method java.lang.String org.apache.iceberg.expressions.Reference::name()" justification: "All subclasses implement name" - - code: "java.method.addedToInterface" - new: "method org.apache.iceberg.actions.MigrateTable org.apache.iceberg.actions.MigrateTable::dropBackup()" - justification: "Adding new functionality to allow for dropping backup table" - code: "java.method.addedToInterface" new: "method java.util.List org.apache.iceberg.Table::statisticsFiles()" justification: "new API method" - code: "java.method.removed" old: "method java.lang.Iterable org.apache.iceberg.Snapshot::addedFiles()" justification: "Deprecations for 1.0 release" + - code: "java.method.removed" + old: "method java.lang.Iterable org.apache.iceberg.Snapshot::deletedFiles()" + justification: "Deprecations for 1.0 release" - code: "java.method.removed" old: "method java.util.List org.apache.iceberg.Snapshot::allManifests()" justification: "Deprecations for 1.0 release" @@ -29,9 +95,6 @@ acceptedBreaks: - code: "java.method.removed" old: "method java.util.List org.apache.iceberg.Snapshot::deleteManifests()" justification: "Deprecations for 1.0 release" - - code: "java.method.removed" - old: "method java.lang.Iterable org.apache.iceberg.Snapshot::deletedFiles()" - justification: "Deprecations for 1.0 release" - code: "java.method.removed" old: "method org.apache.iceberg.OverwriteFiles org.apache.iceberg.OverwriteFiles::validateNoConflictingAppends(org.apache.iceberg.expressions.Expression)" justification: "Deprecations for 1.0 release" @@ -41,9 +104,6 @@ acceptedBreaks: - code: "java.method.removed" old: "method org.apache.iceberg.RowDelta org.apache.iceberg.RowDelta::validateNoConflictingAppends(org.apache.iceberg.expressions.Expression)" justification: "Deprecations for 1.0 release" - - code: "java.method.addedToInterface" - new: "method org.apache.iceberg.TableScan org.apache.iceberg.TableScan::useRef(java.lang.String)" - justification: "Adding table scan APIs to support scanning from refs" release-base-0.13.0: org.apache.iceberg:iceberg-api: - code: "java.class.defaultSerializationChanged" diff --git a/api/src/main/java/org/apache/iceberg/TableScan.java b/api/src/main/java/org/apache/iceberg/TableScan.java index ac70e1f5d890..5d2a1269d633 100644 --- a/api/src/main/java/org/apache/iceberg/TableScan.java +++ b/api/src/main/java/org/apache/iceberg/TableScan.java @@ -45,7 +45,9 @@ public interface TableScan extends Scan * * @return this for method chaining */ - MigrateTable dropBackup(); + default MigrateTable dropBackup() { + throw new UnsupportedOperationException("Dropping a backup is not supported"); + } /** The action result that contains a summary of the execution. */ interface Result { diff --git a/build.gradle b/build.gradle index e5ab7377b162..12ed7017ecbf 100644 --- a/build.gradle +++ b/build.gradle @@ -91,6 +91,7 @@ dependencyRecommendations { } def projectVersion = getProjectVersion() +final REVAPI_PROJECTS = ["iceberg-api", "iceberg-core", "iceberg-parquet", "iceberg-orc", "iceberg-common", "iceberg-data"] allprojects { group = "org.apache.iceberg" @@ -105,6 +106,15 @@ subprojects { apply plugin: 'nebula.dependency-recommender' apply plugin: 'java-library' + if (project.name in REVAPI_PROJECTS) { + apply plugin: 'com.palantir.revapi' + revapi { + oldGroup = project.group + oldName = project.name + oldVersion = "1.0.0" + } + } + configurations { testImplementation.extendsFrom compileOnly @@ -225,14 +235,6 @@ project(':iceberg-bundled-guava') { } project(':iceberg-api') { - apply plugin: 'com.palantir.revapi' - - revapi { - oldGroup = "org.apache.iceberg" - oldName = "iceberg-api" - oldVersion = "1.0.0" - } - dependencies { implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') compileOnly "com.google.errorprone:error_prone_annotations" diff --git a/core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java b/core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java index dd44505e9d39..053ced8d37b7 100644 --- a/core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java +++ b/core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java @@ -79,6 +79,20 @@ public ReplacePartitions validateNoConflictingData() { return this; } + /** + * Validate the current metadata. + * + *

Child operations can override this to add custom validation. + * + * @param currentMetadata current table metadata to validate + * @deprecated Will be removed in 1.2.0, use {@link SnapshotProducer#validate(TableMetadata, + * Snapshot)}. + */ + @Deprecated + public void validate(TableMetadata currentMetadata) { + super.validate(currentMetadata); + } + @Override public void validate(TableMetadata currentMetadata, Snapshot snapshot) { if (validateConflictingData) { @@ -100,6 +114,20 @@ public void validate(TableMetadata currentMetadata, Snapshot snapshot) { } } + /** + * Apply the update's changes to the base table metadata and return the new manifest list. + * + * @param base the base table metadata to apply changes to + * @return a manifest list for the new snapshot. + * @deprecated Will be removed in 1.2.0, use {@link BaseReplacePartitions#apply(TableMetadata, + * Snapshot)}. + */ + @Deprecated + @Override + public List apply(TableMetadata base) { + return super.apply(base); + } + @Override public List apply(TableMetadata base, Snapshot snapshot) { if (dataSpec().fields().size() <= 0) { diff --git a/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java b/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java index 54bf3c6e44c4..28c2c6588f28 100644 --- a/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java +++ b/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java @@ -173,6 +173,20 @@ private ManifestFile copyManifest(ManifestFile manifest) { summaryBuilder); } + /** + * Apply the update's changes to the base table metadata and return the new manifest list. + * + * @param base the base table metadata to apply changes to + * @return a manifest list for the new snapshot. + * @deprecated Will be removed in 1.2.0, use {@link BaseRewriteManifests#apply(TableMetadata, + * Snapshot)}. + */ + @Deprecated + @Override + public List apply(TableMetadata base) { + return super.apply(base); + } + @Override public List apply(TableMetadata base, Snapshot snapshot) { List currentManifests = base.currentSnapshot().dataManifests(ops.io()); diff --git a/core/src/main/java/org/apache/iceberg/BaseTableScan.java b/core/src/main/java/org/apache/iceberg/BaseTableScan.java index 428d20c05173..ba25467f7890 100644 --- a/core/src/main/java/org/apache/iceberg/BaseTableScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseTableScan.java @@ -61,6 +61,16 @@ protected Long snapshotId() { return context().snapshotId(); } + /** + * @return whether column stats are returned. + * @deprecated Will be removed in 1.2.0, use {@link TableScanContext#returnColumnStats()} + * directly. + */ + @Deprecated + protected boolean colStats() { + return context().returnColumnStats(); + } + protected Map options() { return context().options(); } diff --git a/core/src/main/java/org/apache/iceberg/FastAppend.java b/core/src/main/java/org/apache/iceberg/FastAppend.java index 5e5e5128411a..9e7a7e53e795 100644 --- a/core/src/main/java/org/apache/iceberg/FastAppend.java +++ b/core/src/main/java/org/apache/iceberg/FastAppend.java @@ -141,6 +141,19 @@ private ManifestFile copyManifest(ManifestFile manifest) { summaryBuilder); } + /** + * Apply the update's changes to the base table metadata and return the new manifest list. + * + * @param base the base table metadata to apply changes to + * @return a manifest list for the new snapshot. + * @deprecated Will be removed in 1.2.0, use {@link FastAppend#apply(TableMetadata, Snapshot)}. + */ + @Deprecated + @Override + public List apply(TableMetadata base) { + return super.apply(base); + } + @Override public List apply(TableMetadata base, Snapshot snapshot) { List newManifests = Lists.newArrayList(); diff --git a/core/src/main/java/org/apache/iceberg/ManifestReader.java b/core/src/main/java/org/apache/iceberg/ManifestReader.java index 9c4964746b7c..ef2549720a73 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/ManifestReader.java @@ -98,6 +98,38 @@ private String fileClass() { private Evaluator lazyEvaluator = null; private InclusiveMetricsEvaluator lazyMetricsEvaluator = null; + /** + * @deprecated Will be removed in 1.2.0, use {@link ManifestReader#ManifestReader(InputFile, int, + * Map, InheritableMetadata, FileType)}. + */ + @Deprecated + protected ManifestReader( + InputFile file, + Map specsById, + InheritableMetadata inheritableMetadata, + FileType content) { + this.file = file; + this.inheritableMetadata = inheritableMetadata; + this.content = content; + + Map metadata = readMetadata(file); + int specId = TableMetadata.INITIAL_SPEC_ID; + String specProperty = metadata.get("partition-spec-id"); + if (specProperty != null) { + specId = Integer.parseInt(specProperty); + } + + if (specsById != null) { + this.spec = specsById.get(specId); + } else { + Schema schema = SchemaParser.fromJson(metadata.get("schema")); + this.spec = + PartitionSpecParser.fromJsonFields(schema, specId, metadata.get("partition-spec")); + } + + this.fileSchema = new Schema(DataFile.getType(spec.partitionType()).fields()); + } + protected ManifestReader( InputFile file, int specId, @@ -118,6 +150,19 @@ protected ManifestReader( } private > PartitionSpec readPartitionSpec(InputFile inputFile) { + Map metadata = readMetadata(inputFile); + + int specId = TableMetadata.INITIAL_SPEC_ID; + String specProperty = metadata.get("partition-spec-id"); + if (specProperty != null) { + specId = Integer.parseInt(specProperty); + } + + Schema schema = SchemaParser.fromJson(metadata.get("schema")); + return PartitionSpecParser.fromJsonFields(schema, specId, metadata.get("partition-spec")); + } + + private static > Map readMetadata(InputFile inputFile) { Map metadata; try { try (AvroIterable> headerReader = @@ -130,15 +175,7 @@ private > PartitionSpec readPartitionSpec(InputFile inp } catch (IOException e) { throw new RuntimeIOException(e); } - - int specId = TableMetadata.INITIAL_SPEC_ID; - String specProperty = metadata.get("partition-spec-id"); - if (specProperty != null) { - specId = Integer.parseInt(specProperty); - } - - Schema schema = SchemaParser.fromJson(metadata.get("schema")); - return PartitionSpecParser.fromJsonFields(schema, specId, metadata.get("partition-spec")); + return metadata; } public boolean isDeleteManifestReader() { diff --git a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java index e0452e0be239..0d47157dadba 100644 --- a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java @@ -758,6 +758,20 @@ protected Map summary() { return summaryBuilder.build(); } + /** + * Apply the update's changes to the base table metadata and return the new manifest list. + * + * @param base the base table metadata to apply changes to + * @return a manifest list for the new snapshot. + * @deprecated Will be removed in 1.2.0, use {@link MergingSnapshotProducer#apply(TableMetadata, + * Snapshot)}. + */ + @Deprecated + @Override + public List apply(TableMetadata base) { + return super.apply(base); + } + @Override public List apply(TableMetadata base, Snapshot snapshot) { // filter any existing manifests diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index 0e8732ea63f9..92ae95b377b9 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -157,6 +157,20 @@ public ThisT deleteWith(Consumer deleteCallback) { */ protected abstract String operation(); + /** + * Validate the current metadata. + * + *

Child operations can override this to add custom validation. + * + * @param currentMetadata current table metadata to validate + * @deprecated Will be removed in 1.2.0, use {@link SnapshotProducer#validate(TableMetadata, + * Snapshot)}. + */ + @Deprecated + protected void validate(TableMetadata currentMetadata) { + validate(currentMetadata, base.currentSnapshot()); + } + /** * Validate the current metadata. * @@ -167,6 +181,19 @@ public ThisT deleteWith(Consumer deleteCallback) { */ protected void validate(TableMetadata currentMetadata, Snapshot snapshot) {} + /** + * Apply the update's changes to the base table metadata and return the new manifest list. + * + * @param metadataToUpdate the base table metadata to apply changes to + * @return a manifest list for the new snapshot. + * @deprecated Will be removed in 1.2.0, use {@link SnapshotProducer#apply(TableMetadata, + * Snapshot)}. + */ + @Deprecated + protected List apply(TableMetadata metadataToUpdate) { + return apply(metadataToUpdate, base.currentSnapshot()); + } + /** * Apply the update's changes to the given metadata and snapshot. Return the new manifest list. * diff --git a/core/src/main/java/org/apache/iceberg/deletes/Deletes.java b/core/src/main/java/org/apache/iceberg/deletes/Deletes.java index beec06e045d3..7fc118d17a03 100644 --- a/core/src/main/java/org/apache/iceberg/deletes/Deletes.java +++ b/core/src/main/java/org/apache/iceberg/deletes/Deletes.java @@ -83,6 +83,21 @@ public static CloseableIterable markDeleted( }); } + /** + * Returns the remaining rows (the ones that are not deleted), while counting the deleted ones. + * + * @param rows the rows to process + * @param isDeleted a predicate that determines if a row is deleted + * @return the processed rows + * @deprecated Will be removed in 1.2.0, use {@link Deletes#filterDeleted(CloseableIterable, + * Predicate, DeleteCounter)}. + */ + @Deprecated + public static CloseableIterable filterDeleted( + CloseableIterable rows, Predicate isDeleted) { + return filterDeleted(rows, isDeleted, new DeleteCounter()); + } + /** * Returns the remaining rows (the ones that are not deleted), while counting the deleted ones. * diff --git a/core/src/main/java/org/apache/iceberg/hadoop/Util.java b/core/src/main/java/org/apache/iceberg/hadoop/Util.java index 8ff792920af4..ebe862cfcdb0 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/Util.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/Util.java @@ -72,6 +72,14 @@ public static String[] blockLocations(CombinedScanTask task, Configuration conf) return locationSets.toArray(new String[0]); } + /** + * @deprecated Will be removed in 1.2.0, use {@link Util#blockLocations(FileIO, ContentScanTask)}. + */ + @Deprecated + public static String[] blockLocations(FileIO io, CombinedScanTask task) { + return blockLocations(io, (ScanTaskGroup) task); + } + public static String[] blockLocations(FileIO io, ScanTaskGroup taskGroup) { Set locations = Sets.newHashSet(); diff --git a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java index 76f56d353e43..b2c4981db924 100644 --- a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java +++ b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java @@ -269,6 +269,15 @@ public T post( return execute(Method.POST, path, null, body, responseType, headers, errorHandler); } + @Override + public T delete( + String path, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(Method.DELETE, path, null, null, responseType, headers, errorHandler); + } + @Override public T delete( String path, diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTClient.java b/core/src/main/java/org/apache/iceberg/rest/RESTClient.java index 8e6985862663..08bc0c5fa58d 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTClient.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTClient.java @@ -54,11 +54,23 @@ default T delete( T delete( String path, - Map queryParams, Class responseType, Map headers, Consumer errorHandler); + default T delete( + String path, + Map queryParams, + Class responseType, + Map headers, + Consumer errorHandler) { + if (null != queryParams && !queryParams.isEmpty()) { + throw new UnsupportedOperationException("Query params are not supported"); + } + + return delete(path, responseType, headers, errorHandler); + } + default T get( String path, Class responseType, diff --git a/core/src/main/java/org/apache/iceberg/util/PartitionUtil.java b/core/src/main/java/org/apache/iceberg/util/PartitionUtil.java index af2f79c3c6f9..3671bf9bf155 100644 --- a/core/src/main/java/org/apache/iceberg/util/PartitionUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/PartitionUtil.java @@ -22,6 +22,8 @@ import java.util.Map; import java.util.function.BiFunction; import org.apache.iceberg.ContentScanTask; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -29,10 +31,41 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.types.Types.StructType; public class PartitionUtil { private PartitionUtil() {} + /** + * @deprecated Will be removed in 1.2.0, use {@link PartitionUtil#constantsMap(ContentScanTask)}. + */ + @Deprecated + public static Map constantsMap(FileScanTask task) { + return constantsMap((ContentScanTask) task); + } + + /** + * @deprecated Will be removed in 1.2.0, use {@link PartitionUtil#constantsMap(ContentScanTask, + * BiFunction)}. + */ + @Deprecated + public static Map constantsMap( + FileScanTask task, BiFunction convertConstant) { + return constantsMap((ContentScanTask) task, convertConstant); + } + + /** + * @deprecated Will be removed in 1.2.0, use {@link PartitionUtil#constantsMap(ContentScanTask, + * StructType, BiFunction)}. + */ + @Deprecated + public static Map constantsMap( + FileScanTask task, + Types.StructType partitionType, + BiFunction convertConstant) { + return constantsMap((ContentScanTask) task, partitionType, convertConstant); + } + public static Map constantsMap(ContentScanTask task) { return constantsMap(task, null, (type, constant) -> constant); } diff --git a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java index 55623c0d9ed7..6cf07784d492 100644 --- a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java +++ b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java @@ -401,6 +401,15 @@ public T execute( throw new RESTException("Unhandled error: %s", error); } + @Override + public T delete( + String path, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.DELETE, path, null, null, responseType, headers, errorHandler); + } + @Override public T delete( String path,