diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 9c9aca02e478..6c2628cc0026 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -261,235 +261,6 @@ acceptedBreaks: \ T) throws java.io.IOException, com.fasterxml.jackson.core.JacksonException\ \ @ org.apache.iceberg.rest.RESTSerializers.UpdateRequirementDeserializer" justification: "False positive - JacksonException is a subclass of IOException" - - code: "java.method.inheritedMovedToClass" - old: "method ThisT org.apache.iceberg.SnapshotUpdate::toBranch(java.lang.String)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - new: "method org.apache.iceberg.BaseOverwriteFiles org.apache.iceberg.BaseOverwriteFiles::toBranch(java.lang.String)" - justification: "Introducing branch snapshot operations on BaseOverwrite" - - code: "java.method.inheritedMovedToClass" - old: "method ThisT org.apache.iceberg.SnapshotUpdate::toBranch(java.lang.String)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - new: "method org.apache.iceberg.BaseReplacePartitions org.apache.iceberg.BaseReplacePartitions::toBranch(java.lang.String)" - justification: "Introducing branch snapshot operations for BaseReplacePartitions" - - code: "java.method.numberOfParametersChanged" - old: "method org.apache.iceberg.DeleteFileIndex org.apache.iceberg.MergingSnapshotProducer::addedDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.util.PartitionSet)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - new: "method org.apache.iceberg.DeleteFileIndex org.apache.iceberg.MergingSnapshotProducer::addedDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.util.PartitionSet,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method org.apache.iceberg.DeleteFileIndex org.apache.iceberg.MergingSnapshotProducer::addedDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.util.PartitionSet)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - new: "method org.apache.iceberg.DeleteFileIndex org.apache.iceberg.MergingSnapshotProducer::addedDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.util.PartitionSet,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method org.apache.iceberg.DeleteFileIndex org.apache.iceberg.MergingSnapshotProducer::addedDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.util.PartitionSet)\ - \ @ org.apache.iceberg.StreamingDelete" - new: "method org.apache.iceberg.DeleteFileIndex org.apache.iceberg.MergingSnapshotProducer::addedDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.util.PartitionSet,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateAddedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDataFilesExist(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.CharSequenceSet, boolean, org.apache.iceberg.expressions.Expression)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDataFilesExist(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.CharSequenceSet, boolean, org.apache.iceberg.expressions.Expression,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDataFilesExist(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.CharSequenceSet, boolean, org.apache.iceberg.expressions.Expression)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDataFilesExist(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.CharSequenceSet, boolean, org.apache.iceberg.expressions.Expression,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDataFilesExist(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.CharSequenceSet, boolean, org.apache.iceberg.expressions.Expression)\ - \ @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDataFilesExist(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.CharSequenceSet, boolean, org.apache.iceberg.expressions.Expression,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateDeletedDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression) @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet) @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.util.PartitionSet, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, java.lang.Iterable) @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, java.lang.Iterable, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, java.lang.Iterable) @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, java.lang.Iterable, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, java.lang.Iterable) @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, java.lang.Iterable, org.apache.iceberg.Snapshot)\ - \ @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, java.lang.Iterable)\ - \ @ org.apache.iceberg.BaseOverwriteFiles" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, java.lang.Iterable,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.BaseOverwriteFiles" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, java.lang.Iterable)\ - \ @ org.apache.iceberg.BaseReplacePartitions" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, java.lang.Iterable,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.BaseReplacePartitions" - justification: "Passing parent snapshot for supporting operations on branch" - - code: "java.method.numberOfParametersChanged" - old: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, java.lang.Iterable)\ - \ @ org.apache.iceberg.StreamingDelete" - new: "method void org.apache.iceberg.MergingSnapshotProducer::validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata,\ - \ java.lang.Long, org.apache.iceberg.expressions.Expression, java.lang.Iterable,\ - \ org.apache.iceberg.Snapshot) @ org.apache.iceberg.StreamingDelete" - justification: "Passing parent snapshot for supporting operations on branch" - code: "java.method.removed" old: "method org.apache.iceberg.io.CloseableIterable org.apache.iceberg.deletes.Deletes::filterDeleted(org.apache.iceberg.io.CloseableIterable,\ \ java.util.function.Predicate)" diff --git a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java index 7e212c16401b..57df521b512a 100644 --- a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java @@ -268,6 +268,20 @@ private ManifestFile copyManifest(ManifestFile manifest) { appendedManifestsSummary); } + /** + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param partitionSet a set of partitions to filter new conflicting data files + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#validateAddedDataFiles(TableMetadata, Long, PartitionSet, + * Snapshot)} instead + */ + @Deprecated + protected void validateAddedDataFiles( + TableMetadata base, Long startingSnapshotId, PartitionSet partitionSet) { + validateAddedDataFiles(base, startingSnapshotId, partitionSet, base.currentSnapshot()); + } + /** * Validates that no files matching given partitions have been added to the table since a starting * snapshot. @@ -275,6 +289,7 @@ private ManifestFile copyManifest(ManifestFile manifest) { * @param base table metadata to validate * @param startingSnapshotId id of the snapshot current at the start of the operation * @param partitionSet a set of partitions to filter new conflicting data files + * @param parent ending snapshot on the lineage being validated */ protected void validateAddedDataFiles( TableMetadata base, Long startingSnapshotId, PartitionSet partitionSet, Snapshot parent) { @@ -296,6 +311,21 @@ protected void validateAddedDataFiles( } } + /** + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param conflictDetectionFilter an expression used to find new conflicting data files + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#validateAddedDataFiles(TableMetadata, Long, Expression, Snapshot)} + * instead + */ + @Deprecated + protected void validateAddedDataFiles( + TableMetadata base, Long startingSnapshotId, Expression conflictDetectionFilter) { + validateAddedDataFiles( + base, startingSnapshotId, conflictDetectionFilter, base.currentSnapshot()); + } + /** * Validates that no files matching a filter have been added to the table since a starting * snapshot. @@ -328,13 +358,14 @@ protected void validateAddedDataFiles( } /** - * Returns an iterable of files matching a filter have been added to the table since a starting + * Returns an iterable of files matching a filter have been added to a branch since a starting * snapshot. * * @param base table metadata to validate * @param startingSnapshotId id of the snapshot current at the start of the operation * @param dataFilter an expression used to find new data files * @param partitionSet a set of partitions to find new data files + * @param parent ending snapshot of the branch */ private CloseableIterable> addedDataFiles( TableMetadata base, @@ -385,6 +416,7 @@ private CloseableIterable> addedDataFiles( * @param base table metadata to validate * @param startingSnapshotId id of the snapshot current at the start of the operation * @param dataFiles data files to validate have no new row deletes + * @param parent ending snapshot on the branch being validated */ protected void validateNoNewDeletesForDataFiles( TableMetadata base, Long startingSnapshotId, Iterable dataFiles, Snapshot parent) { @@ -392,6 +424,28 @@ protected void validateNoNewDeletesForDataFiles( base, startingSnapshotId, null, dataFiles, newFilesSequenceNumber != null, parent); } + /** + * Validates that no new delete files that must be applied to the given data files have been added + * to the table since a starting snapshot. + * + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param dataFiles data files to validate have no new row deletes + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#validateNoNewDeletesForDataFiles} instead + */ + @Deprecated + protected void validateNoNewDeletesForDataFiles( + TableMetadata base, Long startingSnapshotId, Iterable dataFiles) { + validateNoNewDeletesForDataFiles( + base, + startingSnapshotId, + null, + dataFiles, + newFilesSequenceNumber != null, + base.currentSnapshot()); + } + /** * Validates that no new delete files that must be applied to the given data files have been added * to the table since a starting snapshot. @@ -400,6 +454,28 @@ protected void validateNoNewDeletesForDataFiles( * @param startingSnapshotId id of the snapshot current at the start of the operation * @param dataFilter a data filter * @param dataFiles data files to validate have no new row deletes + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#validateNoNewDeletesForDataFiles} instead + */ + @Deprecated + protected void validateNoNewDeletesForDataFiles( + TableMetadata base, + Long startingSnapshotId, + Expression dataFilter, + Iterable dataFiles) { + validateNoNewDeletesForDataFiles( + base, startingSnapshotId, dataFilter, dataFiles, base.currentSnapshot()); + } + + /** + * Validates that no new delete files that must be applied to the given data files have been added + * to the table since a starting snapshot. + * + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param dataFilter a data filter + * @param dataFiles data files to validate have no new row deletes + * @param parent ending snapshot on the branch being validated */ protected void validateNoNewDeletesForDataFiles( TableMetadata base, @@ -427,6 +503,7 @@ protected void validateNoNewDeletesForDataFiles( * @param dataFilter a data filter * @param dataFiles data files to validate have no new row deletes * @param ignoreEqualityDeletes whether equality deletes should be ignored in validation + * @param parent ending snapshot on the branch being validated */ private void validateNoNewDeletesForDataFiles( TableMetadata base, @@ -469,6 +546,23 @@ private void validateNoNewDeletesForDataFiles( * @param base table metadata to validate * @param startingSnapshotId id of the snapshot current at the start of the operation * @param dataFilter an expression used to find new conflicting delete files + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#validateNoNewDeleteFiles} instead + */ + @Deprecated + protected void validateNoNewDeleteFiles( + TableMetadata base, Long startingSnapshotId, Expression dataFilter) { + validateNoNewDeleteFiles(base, startingSnapshotId, dataFilter, base.currentSnapshot()); + } + + /** + * Validates that no delete files matching a filter have been added to the table since a starting + * snapshot. + * + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param dataFilter an expression used to find new conflicting delete files + * @param parent ending snapshot on the branch being validated */ protected void validateNoNewDeleteFiles( TableMetadata base, Long startingSnapshotId, Expression dataFilter, Snapshot parent) { @@ -487,6 +581,24 @@ protected void validateNoNewDeleteFiles( * @param base table metadata to validate * @param startingSnapshotId id of the snapshot current at the start of the operation * @param partitionSet a partition set used to find new conflicting delete files + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#validateNoNewDeleteFiles(TableMetadata, Long, PartitionSet, + * Snapshot)} instead + */ + @Deprecated + protected void validateNoNewDeleteFiles( + TableMetadata base, Long startingSnapshotId, PartitionSet partitionSet) { + validateNoNewDeleteFiles(base, startingSnapshotId, partitionSet, base.currentSnapshot()); + } + + /** + * Validates that no delete files matching a partition set have been added to the table since a + * starting snapshot. + * + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param partitionSet a partition set used to find new conflicting delete files + * @param parent ending snapshot on the branch being validated */ protected void validateNoNewDeleteFiles( TableMetadata base, Long startingSnapshotId, PartitionSet partitionSet, Snapshot parent) { @@ -506,6 +618,28 @@ protected void validateNoNewDeleteFiles( * @param startingSnapshotId id of the snapshot current at the start of the operation * @param dataFilter an expression used to find delete files * @param partitionSet a partition set used to find delete files + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#addedDeleteFiles(TableMetadata, Long, Expression, PartitionSet, + * Snapshot)} instead + */ + @Deprecated + protected DeleteFileIndex addedDeleteFiles( + TableMetadata base, + Long startingSnapshotId, + Expression dataFilter, + PartitionSet partitionSet) { + return addedDeleteFiles( + base, startingSnapshotId, dataFilter, partitionSet, base.currentSnapshot()); + } + + /** + * Returns matching delete files have been added to the table since a starting snapshot. + * + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param dataFilter an expression used to find delete files + * @param partitionSet a partition set used to find delete files + * @param parent parent snapshot of the branch */ protected DeleteFileIndex addedDeleteFiles( TableMetadata base, @@ -540,6 +674,24 @@ protected DeleteFileIndex addedDeleteFiles( * @param base table metadata to validate * @param startingSnapshotId id of the snapshot current at the start of the operation * @param dataFilter an expression used to find deleted data files + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#validateDeletedDataFiles(TableMetadata, Long, Expression, + * Snapshot)} instead + */ + @Deprecated + protected void validateDeletedDataFiles( + TableMetadata base, Long startingSnapshotId, Expression dataFilter) { + validateDeletedDataFiles(base, startingSnapshotId, dataFilter, base.currentSnapshot()); + } + + /** + * Validates that no files matching a filter have been deleted from the table since a starting + * snapshot. + * + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param dataFilter an expression used to find deleted data files + * @param parent ending snapshot on the branch being validated */ protected void validateDeletedDataFiles( TableMetadata base, Long startingSnapshotId, Expression dataFilter, Snapshot parent) { @@ -568,6 +720,24 @@ protected void validateDeletedDataFiles( * @param base table metadata to validate * @param startingSnapshotId id of the snapshot current at the start of the operation * @param partitionSet a partition set used to find deleted data files + * @deprecated will be removed in 1.3.0; use {@link + * MergingSnapshotProducer#validateNoNewDeleteFiles(TableMetadata, Long, PartitionSet, + * Snapshot)} instead + */ + @Deprecated + protected void validateDeletedDataFiles( + TableMetadata base, Long startingSnapshotId, PartitionSet partitionSet) { + validateDeletedDataFiles(base, startingSnapshotId, partitionSet, base.currentSnapshot()); + } + + /** + * Validates that no files matching a filter have been deleted from the table since a starting + * snapshot. + * + * @param base table metadata to validate + * @param startingSnapshotId id of the snapshot current at the start of the operation + * @param partitionSet a partition set used to find deleted data files + * @param parent ending snapshot on the branch being validated */ protected void validateDeletedDataFiles( TableMetadata base, Long startingSnapshotId, PartitionSet partitionSet, Snapshot parent) { @@ -597,6 +767,7 @@ protected void validateDeletedDataFiles( * @param startingSnapshotId id of the snapshot current at the start of the operation * @param dataFilter an expression used to find deleted data files * @param partitionSet a set of partitions to find deleted data files + * @param parent ending snapshot on the branch being validated */ private CloseableIterable> deletedDataFiles( TableMetadata base, @@ -675,6 +846,27 @@ private DeleteFileIndex buildDeleteFileIndex( return builder.build(); } + @Deprecated + /** + * @deprecated will be removed in 1.3.0 use {@link + * MergingSnapshotProducer#validateDataFilesExist(TableMetadata, Long, CharSequenceSet, + * boolean, Expression, Snapshot)} instead. + */ + protected void validateDataFilesExist( + TableMetadata base, + Long startingSnapshotId, + CharSequenceSet requiredDataFiles, + boolean skipDeletes, + Expression conflictDetectionFilter) { + validateDataFilesExist( + base, + startingSnapshotId, + requiredDataFiles, + skipDeletes, + conflictDetectionFilter, + base.currentSnapshot()); + } + @SuppressWarnings("CollectionUndefinedEquality") protected void validateDataFilesExist( TableMetadata base,