-
Notifications
You must be signed in to change notification settings - Fork 3k
Core, API: Append to branch Impl #5010
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
9e43889
e6ea5f6
b43b80b
9cb39c2
ba341ec
d1ca432
0d79ef0
0e2f690
bcfb7a8
495c0f3
f7464eb
8d80888
ca774b2
fba4643
5456a3a
7ecc325
637b2e8
40e5e62
13a8e73
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -114,6 +114,21 @@ public FastAppend appendManifest(ManifestFile manifest) { | |
| return this; | ||
| } | ||
|
|
||
| @Override | ||
| public FastAppend toBranch(String branch) { | ||
| Preconditions.checkArgument(branch != null, "branch cannot be null"); | ||
| if (ops.current().ref(branch) == null) { | ||
| super.createNewRef(branch); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This should be handled by |
||
| } | ||
|
|
||
| Preconditions.checkArgument(ops.current() | ||
| .ref(branch).type() | ||
| .equals(SnapshotRefType.BRANCH), | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does this check need to be split across 3 lines? Or is that autoformatting doing it? |
||
| "%s is not a ref to type branch", branch); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Also now that we're creating a branch if it doesn't exist, on commit, we should do a null check. It's okay if the branch is null, since we'll create it. if it's not null, it must be an actual branch. I think this should also reside in snapshot producer as well. |
||
| setTargetBranch(branch); | ||
| return this; | ||
| } | ||
|
|
||
| private ManifestFile copyManifest(ManifestFile manifest) { | ||
| TableMetadata current = ops.current(); | ||
| InputFile toCopy = ops.io().newInputFile(manifest.path()); | ||
|
|
@@ -125,6 +140,8 @@ private ManifestFile copyManifest(ManifestFile manifest) { | |
| @Override | ||
| public List<ManifestFile> apply(TableMetadata base) { | ||
| List<ManifestFile> newManifests = Lists.newArrayList(); | ||
| Snapshot current = base.ref(getTargetBranch()) != null ? | ||
| base.snapshot(base.ref(getTargetBranch()).snapshotId()) : base.currentSnapshot(); | ||
|
|
||
| try { | ||
| ManifestFile manifest = writeManifest(); | ||
|
|
@@ -140,8 +157,8 @@ public List<ManifestFile> apply(TableMetadata base) { | |
| manifest -> GenericManifestFile.copyOf(manifest).withSnapshotId(snapshotId()).build()); | ||
| Iterables.addAll(newManifests, appendManifestsWithMetadata); | ||
|
|
||
| if (base.currentSnapshot() != null) { | ||
| newManifests.addAll(base.currentSnapshot().allManifests(ops.io())); | ||
| if (current != null) { | ||
| newManifests.addAll(current.allManifests(ops.io())); | ||
| } | ||
|
|
||
| return newManifests; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -88,6 +88,7 @@ public void accept(String file) { | |
| private Consumer<String> deleteFunc = defaultDelete; | ||
|
|
||
| private ExecutorService workerPool = ThreadPools.getWorkerPool(); | ||
| private String targetBranch = SnapshotRef.MAIN_BRANCH; | ||
|
|
||
| protected SnapshotProducer(TableOperations ops) { | ||
| this.ops = ops; | ||
|
|
@@ -116,6 +117,38 @@ public ThisT scanManifestsWith(ExecutorService executorService) { | |
| return self(); | ||
| } | ||
|
|
||
| @Override | ||
| public ThisT toBranch(String branch) { | ||
| throw new UnsupportedOperationException("Performing operations on a branch is currently not supported"); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: the error wording looks a bit confusing to me. Are we planning to support the
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @dimas-b , In the PR i have implemented the
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the clarification, @namrathamyske ! Would you mind updating the message to something like |
||
| } | ||
|
|
||
| /*** | ||
| * Will be used by snapshot producer operations to create a new ref if an invalid branch is passed | ||
| * @param branch ref name on which operation is to performed | ||
| */ | ||
| protected void createNewRef(String branch) { | ||
| SnapshotRef branchRef = SnapshotRef.branchBuilder(this.current().currentSnapshot().snapshotId()).build(); | ||
| TableMetadata.Builder updatedBuilder = TableMetadata.buildFrom(this.current()); | ||
| updatedBuilder.setRef(branch, branchRef); | ||
| ops.commit(ops.current(), updatedBuilder.build()); | ||
| } | ||
|
|
||
| /*** | ||
| * A setter for the target branch on which snapshot producer operation should be performed | ||
| * @param branch to set as target branch | ||
| */ | ||
| protected void setTargetBranch(String branch) { | ||
| this.targetBranch = branch; | ||
| } | ||
|
|
||
| /*** | ||
| * A getter for the target branch on which snapshot producer operation should be performed | ||
| * @return target branch | ||
| */ | ||
| protected String getTargetBranch() { | ||
| return targetBranch; | ||
| } | ||
|
|
||
| protected ExecutorService workerPool() { | ||
| return this.workerPool; | ||
| } | ||
|
|
@@ -167,8 +200,7 @@ protected void validate(TableMetadata currentMetadata) { | |
| @Override | ||
| public Snapshot apply() { | ||
| refresh(); | ||
| Long parentSnapshotId = base.currentSnapshot() != null ? | ||
| base.currentSnapshot().snapshotId() : null; | ||
| Long parentSnapshotId = base.ref(targetBranch) != null ? base.ref(targetBranch).snapshotId() : null; | ||
| long sequenceNumber = base.nextSequenceNumber(); | ||
|
|
||
| // run validations from the child operation | ||
|
|
@@ -298,11 +330,11 @@ public void commit() { | |
| TableMetadata.Builder update = TableMetadata.buildFrom(base); | ||
| if (base.snapshot(newSnapshot.snapshotId()) != null) { | ||
| // this is a rollback operation | ||
| update.setBranchSnapshot(newSnapshot.snapshotId(), SnapshotRef.MAIN_BRANCH); | ||
| update.setBranchSnapshot(newSnapshot.snapshotId(), targetBranch); | ||
| } else if (stageOnly) { | ||
| update.addSnapshot(newSnapshot); | ||
| } else { | ||
| update.setBranchSnapshot(newSnapshot, SnapshotRef.MAIN_BRANCH); | ||
| update.setBranchSnapshot(newSnapshot, targetBranch); | ||
| } | ||
|
|
||
| TableMetadata updated = update.build(); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Message should be
Invalid branch name: null