-
Notifications
You must be signed in to change notification settings - Fork 3.4k
Core, API: Performing operations on a snapshot branch ref #4926
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
9e43889
0e2f690
bcfb7a8
8d80888
637b2e8
c87df9a
9e98792
9c1a2d2
4772047
790613d
6948eac
3a60283
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -60,4 +60,10 @@ public interface SnapshotUpdate<ThisT> extends PendingUpdate<Snapshot> { | |
| * @return this for method chaining | ||
| */ | ||
| ThisT scanManifestsWith(ExecutorService executorService); | ||
|
|
||
| /** | ||
| * Perform operations on a particular branch | ||
| * @param branch which is name of SnapshotRef of type branch. | ||
| */ | ||
| ThisT toBranch(String branch); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This needs a default that throws |
||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -88,6 +88,7 @@ public void accept(String file) { | |
| private Consumer<String> deleteFunc = defaultDelete; | ||
|
|
||
|
namrathamyske marked this conversation as resolved.
namrathamyske marked this conversation as resolved.
|
||
| private ExecutorService workerPool = ThreadPools.getWorkerPool(); | ||
| private String targetBranch = SnapshotRef.MAIN_BRANCH; | ||
|
|
||
| protected SnapshotProducer(TableOperations ops) { | ||
| this.ops = ops; | ||
|
|
@@ -116,6 +117,38 @@ public ThisT scanManifestsWith(ExecutorService executorService) { | |
| return self(); | ||
| } | ||
|
|
||
| @Override | ||
| public ThisT toBranch(String branch) { | ||
| throw new UnsupportedOperationException("Performing operations on a branch is currently not supported"); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor: I think this is a good time to use throw new UnsupportedOperationException(Sting.format(
"Cannot commit to branch %s: %s does not support branch commits",
branch,
this.getClass().getName()); |
||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This should be in the interface, not here.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. good point, fixed |
||
|
|
||
| /*** | ||
| * Will be used by snapshot producer operations to create a new ref if an invalid branch is passed | ||
| * @param branch ref name on which operation is to performed | ||
| */ | ||
| protected void createNewRef(String branch) { | ||
| SnapshotRef branchRef = SnapshotRef.branchBuilder(this.current().currentSnapshot().snapshotId()).build(); | ||
| TableMetadata.Builder updatedBuilder = TableMetadata.buildFrom(this.current()); | ||
| updatedBuilder.setRef(branch, branchRef); | ||
| ops.commit(ops.current(), updatedBuilder.build()); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Committing to a branch while creating it should not commit twice. Instead, the commit should add a snapshot and then create a branch pointing to that snapshot.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah if we're creating a new branch, the commit for the snapshot producer needs to encapsulate the producing of the snapshot and the branch creation |
||
| } | ||
|
|
||
| /*** | ||
| * A setter for the target branch on which snapshot producer operation should be performed | ||
| * @param branch to set as target branch | ||
| */ | ||
| protected void setTargetBranch(String branch) { | ||
| this.targetBranch = branch; | ||
| } | ||
|
|
||
| /*** | ||
| * A getter for the target branch on which snapshot producer operation should be performed | ||
| * @return target branch | ||
| */ | ||
| protected String getTargetBranch() { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Iceberg typically does not use Also, what is the purpose of this method? None of the other configurations are exposed back to the caller. Do we need this at all?
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I see from #5010 that this is used in each child implementation to get the parent snapshot. I think it would be better for maintainability and future implementations if the |
||
| return targetBranch; | ||
| } | ||
|
|
||
| protected ExecutorService workerPool() { | ||
| return this.workerPool; | ||
| } | ||
|
|
@@ -167,8 +200,7 @@ protected void validate(TableMetadata currentMetadata) { | |
| @Override | ||
| public Snapshot apply() { | ||
| refresh(); | ||
| Long parentSnapshotId = base.currentSnapshot() != null ? | ||
| base.currentSnapshot().snapshotId() : null; | ||
| Long parentSnapshotId = base.ref(targetBranch) != null ? base.ref(targetBranch).snapshotId() : null; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This should be the existing branch state, the current snapshot (if creating the branch), or null. |
||
| long sequenceNumber = base.nextSequenceNumber(); | ||
|
|
||
| // run validations from the child operation | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This needs to pass the parent snapshot ID into
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same thing with
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @rdblue My thought process is to check the whether starting snapshot ID is an ancestor of parent snapshot. If not then throw invalid starting snapshot ID exception. Else we should go ahead with checking between starting snapshot ID & parent ID.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. https://github.com/namrathamyske/iceberg/pull/1/files should take care of what @rdblue is talking about here which if I'm not mistaken is more about a cleaner abstraction for applying a change to a specific snapshot, and running a validation to a specific snapshot which will be needed when we implement branch support for the other producers. |
||
|
|
@@ -298,11 +330,11 @@ public void commit() { | |
| TableMetadata.Builder update = TableMetadata.buildFrom(base); | ||
| if (base.snapshot(newSnapshot.snapshotId()) != null) { | ||
| // this is a rollback operation | ||
| update.setBranchSnapshot(newSnapshot.snapshotId(), SnapshotRef.MAIN_BRANCH); | ||
|
namrathamyske marked this conversation as resolved.
Outdated
|
||
| update.setBranchSnapshot(newSnapshot.snapshotId(), targetBranch); | ||
| } else if (stageOnly) { | ||
| update.addSnapshot(newSnapshot); | ||
| } else { | ||
| update.setBranchSnapshot(newSnapshot, SnapshotRef.MAIN_BRANCH); | ||
| update.setBranchSnapshot(newSnapshot, targetBranch); | ||
| } | ||
|
|
||
| TableMetadata updated = update.build(); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -478,4 +478,13 @@ public void testIncludedPartitionSummaryLimit() { | |
| String changedPartitions = table.currentSnapshot().summary().get(SnapshotSummary.CHANGED_PARTITION_COUNT_PROP); | ||
| Assert.assertEquals("Should set changed partition count", "2", changedPartitions); | ||
| } | ||
|
|
||
| @Test(expected = UnsupportedOperationException.class) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. To test exceptions, we use either |
||
| public void testAppendToBranch() throws UnsupportedOperationException { | ||
| table.newFastAppend() | ||
| .appendFile(FILE_A) | ||
|
namrathamyske marked this conversation as resolved.
Outdated
|
||
| .commit(); | ||
| table.manageSnapshots().createBranch("ref", table.currentSnapshot().snapshotId()).commit(); | ||
| table.newFastAppend().appendFile(FILE_B).toBranch("ref").commit(); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could this implement branch commits for
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| } | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.