-
Notifications
You must be signed in to change notification settings - Fork 3k
Core: Delegate bulk and prefix operations in ResolvingFileIO #8169
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
3ac5713
2aace57
2d129e9
df3e850
2009bd2
19230d5
9d73070
f7b4530
f6ea325
d4c591f
8e6d9a1
839da9d
065dd0b
076aa43
ce32d5c
109d385
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ | |
| package org.apache.iceberg.io; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.Iterator; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
|
|
@@ -30,15 +31,22 @@ | |
| import org.apache.iceberg.hadoop.SerializableConfiguration; | ||
| import org.apache.iceberg.relocated.com.google.common.base.Joiner; | ||
| import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; | ||
| import org.apache.iceberg.relocated.com.google.common.collect.Iterators; | ||
| import org.apache.iceberg.relocated.com.google.common.collect.Lists; | ||
| import org.apache.iceberg.relocated.com.google.common.collect.Maps; | ||
| import org.apache.iceberg.relocated.com.google.common.collect.PeekingIterator; | ||
| import org.apache.iceberg.util.SerializableMap; | ||
| import org.apache.iceberg.util.SerializableSupplier; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| /** FileIO implementation that uses location scheme to choose the correct FileIO implementation. */ | ||
| public class ResolvingFileIO implements FileIO, HadoopConfigurable { | ||
| /** | ||
| * FileIO implementation that uses location scheme to choose the correct FileIO implementation. | ||
| * Delegate FileIO implementations should support the mixin interfaces {@link | ||
| * SupportsPrefixOperations} and {@link SupportsBulkOperations}. | ||
| */ | ||
| public class ResolvingFileIO | ||
| implements FileIO, SupportsPrefixOperations, SupportsBulkOperations, HadoopConfigurable { | ||
| private static final Logger LOG = LoggerFactory.getLogger(ResolvingFileIO.class); | ||
| private static final String FALLBACK_IMPL = "org.apache.iceberg.hadoop.HadoopFileIO"; | ||
| private static final String S3_FILE_IO_IMPL = "org.apache.iceberg.aws.s3.S3FileIO"; | ||
|
|
@@ -85,6 +93,44 @@ public void deleteFile(String location) { | |
| io(location).deleteFile(location); | ||
| } | ||
|
|
||
| @Override | ||
| public void deleteFiles(Iterable<String> pathsToDelete) throws BulkDeletionFailureException { | ||
| // peek at the first element to determine the type of FileIO | ||
| Iterator<String> originalIterator = pathsToDelete.iterator(); | ||
| if (!originalIterator.hasNext()) { | ||
| return; | ||
| } | ||
|
|
||
| PeekingIterator<String> iterator = Iterators.peekingIterator(originalIterator); | ||
| FileIO fileIO = io(iterator.peek()); | ||
|
||
| if (!(fileIO instanceof SupportsPrefixOperations)) { | ||
|
||
| throw new UnsupportedOperationException( | ||
| "FileIO doesn't support bulk operations: " + fileIO.getClass().getName()); | ||
| } | ||
|
|
||
| ((SupportsBulkOperations) fileIO).deleteFiles(() -> iterator); | ||
|
||
| } | ||
|
|
||
| @Override | ||
| public Iterable<FileInfo> listPrefix(String prefix) { | ||
| FileIO fileIO = io(prefix); | ||
| if (!(fileIO instanceof SupportsPrefixOperations)) { | ||
|
||
| throw new UnsupportedOperationException( | ||
| "FileIO doesn't support prefix operations: " + fileIO.getClass().getName()); | ||
| } | ||
| return ((SupportsPrefixOperations) fileIO).listPrefix(prefix); | ||
| } | ||
|
|
||
| @Override | ||
| public void deletePrefix(String prefix) { | ||
RussellSpitzer marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| FileIO fileIO = io(prefix); | ||
| if (!(fileIO instanceof SupportsPrefixOperations)) { | ||
| throw new UnsupportedOperationException( | ||
| "FileIO doesn't support prefix operations: " + fileIO.getClass().getName()); | ||
| } | ||
| ((SupportsPrefixOperations) fileIO).deletePrefix(prefix); | ||
| } | ||
|
|
||
| @Override | ||
| public Map<String, String> properties() { | ||
| return properties.immutableMap(); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If we want to make this a requirement we should probably just add these as perquisites in the init code?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That makes sense to me, I'll add that.