-
Notifications
You must be signed in to change notification settings - Fork 3k
Add UpdateSnapshotReference #3612
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
abe9a7d
a29e31b
41edb56
e7d993b
e971b7a
e0cc4ec
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,180 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.iceberg; | ||
|
|
||
| import java.util.Objects; | ||
| import org.apache.iceberg.exceptions.ValidationException; | ||
| import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; | ||
|
|
||
| /** | ||
| * User-defined information of a named snapshot | ||
| */ | ||
| public class SnapshotReference { | ||
|
|
||
| private final long snapshotId; | ||
| private final SnapshotReferenceType type; | ||
| private final Integer minSnapshotsToKeep; | ||
| private final Long maxSnapshotAgeMs; | ||
| private final Long maxRefAgeMs; | ||
|
|
||
| private SnapshotReference( | ||
| long snapshotId, | ||
| SnapshotReferenceType type, | ||
| Integer minSnapshotsToKeep, | ||
| Long maxSnapshotAgeMs, | ||
| Long maxRefAgeMs) { | ||
| this.snapshotId = snapshotId; | ||
| this.type = type; | ||
| this.minSnapshotsToKeep = minSnapshotsToKeep; | ||
| this.maxSnapshotAgeMs = maxSnapshotAgeMs; | ||
| this.maxRefAgeMs = maxRefAgeMs; | ||
| } | ||
|
|
||
| public long snapshotId() { | ||
| return snapshotId; | ||
| } | ||
|
|
||
| public SnapshotReferenceType type() { | ||
| return type; | ||
| } | ||
|
|
||
| public Integer minSnapshotsToKeep() { | ||
| return minSnapshotsToKeep; | ||
| } | ||
|
|
||
| public Long maxSnapshotAgeMs() { | ||
| return maxSnapshotAgeMs; | ||
| } | ||
|
|
||
| public Long maxRefAgeMs() { | ||
| return maxRefAgeMs; | ||
| } | ||
|
|
||
| public static Builder builderForTag(long snapshotId) { | ||
| return builderFor(snapshotId, SnapshotReferenceType.TAG); | ||
| } | ||
|
|
||
| public static Builder builderForBranch(long snapshotId) { | ||
| return builderFor(snapshotId, SnapshotReferenceType.BRANCH); | ||
| } | ||
|
|
||
| public static Builder builderFrom(SnapshotReference ref) { | ||
| return new Builder(ref.type()) | ||
| .snapshotId(ref.snapshotId()) | ||
| .minSnapshotsToKeep(ref.minSnapshotsToKeep()) | ||
| .maxSnapshotAgeMs(ref.maxSnapshotAgeMs()) | ||
| .maxRefAgeMs(ref.maxRefAgeMs()); | ||
| } | ||
|
|
||
| public static Builder builderFor(long snapshotId, SnapshotReferenceType type) { | ||
| return new Builder(type).snapshotId(snapshotId); | ||
| } | ||
|
|
||
| public static class Builder { | ||
|
|
||
| private final SnapshotReferenceType type; | ||
|
|
||
| private Long snapshotId; | ||
| private Integer minSnapshotsToKeep; | ||
| private Long maxSnapshotAgeMs; | ||
| private Long maxRefAgeMs; | ||
|
|
||
| Builder(SnapshotReferenceType type) { | ||
| ValidationException.check(type != null, "Snapshot reference type must not be null"); | ||
| this.type = type; | ||
| } | ||
|
|
||
| public Builder snapshotId(long id) { | ||
| this.snapshotId = id; | ||
| return this; | ||
| } | ||
|
|
||
| public Builder minSnapshotsToKeep(Integer value) { | ||
| this.minSnapshotsToKeep = value; | ||
| return this; | ||
| } | ||
|
|
||
| public Builder maxSnapshotAgeMs(Long value) { | ||
| this.maxSnapshotAgeMs = value; | ||
| return this; | ||
| } | ||
|
|
||
| public Builder maxRefAgeMs(Long value) { | ||
| this.maxRefAgeMs = value; | ||
| return this; | ||
| } | ||
|
|
||
| public SnapshotReference build() { | ||
| if (type.equals(SnapshotReferenceType.TAG)) { | ||
| ValidationException.check(minSnapshotsToKeep == null, | ||
| "TAG type snapshot reference does not support setting minSnapshotsToKeep"); | ||
| ValidationException.check(maxSnapshotAgeMs == null, | ||
| "TAG type snapshot reference does not support setting maxSnapshotAgeMs"); | ||
| } else { | ||
| if (minSnapshotsToKeep != null) { | ||
| ValidationException.check(minSnapshotsToKeep > 0, | ||
| "Min snapshots to keep must be greater than 0"); | ||
| } | ||
|
|
||
| if (maxSnapshotAgeMs != null) { | ||
| ValidationException.check(maxSnapshotAgeMs > 0, "Max snapshot age must be greater than 0"); | ||
| } | ||
| } | ||
|
|
||
| if (maxRefAgeMs != null) { | ||
| ValidationException.check(maxRefAgeMs > 0, "Max reference age must be greater than 0"); | ||
| } | ||
|
|
||
| return new SnapshotReference(snapshotId, type, minSnapshotsToKeep, maxSnapshotAgeMs, maxRefAgeMs); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object o) { | ||
| if (this == o) { | ||
| return true; | ||
| } else if (o == null || getClass() != o.getClass()) { | ||
| return false; | ||
| } | ||
|
|
||
| SnapshotReference that = (SnapshotReference) o; | ||
| return snapshotId == that.snapshotId && | ||
| type == that.type && | ||
| Objects.equals(minSnapshotsToKeep, that.minSnapshotsToKeep) && | ||
| Objects.equals(maxSnapshotAgeMs, that.maxSnapshotAgeMs) && | ||
| Objects.equals(maxRefAgeMs, that.maxRefAgeMs); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(snapshotId, type, minSnapshotsToKeep, maxSnapshotAgeMs, maxRefAgeMs); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return MoreObjects.toStringHelper(this) | ||
| .add("snapshotId", snapshotId) | ||
| .add("type", type) | ||
| .add("minSnapshotsToKeep", minSnapshotsToKeep) | ||
| .add("maxSnapshotAgeMs", maxSnapshotAgeMs) | ||
| .add("maxRefAgeMs", maxRefAgeMs) | ||
| .toString(); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.iceberg; | ||
|
|
||
| enum SnapshotReferenceType { | ||
| BRANCH, | ||
| TAG | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.iceberg; | ||
|
|
||
| import java.util.Map; | ||
| import org.apache.iceberg.exceptions.CommitFailedException; | ||
|
|
||
| /** | ||
| * API for updating snapshot reference. | ||
| * <p> | ||
| * When committing, these changes will be applied to the current table metadata. Commit conflicts | ||
| * will not be resolved and will result in a {@link CommitFailedException}. | ||
| */ | ||
| public interface UpdateSnapshotReference extends PendingUpdate<Map<String, SnapshotReference>> { | ||
|
|
||
| /** | ||
| * remove snapshotReference. | ||
| * | ||
| * @param name name of snapshot reference | ||
| * @return this | ||
| * @throws IllegalArgumentException If there is no such snapshot reference named name | ||
| */ | ||
| UpdateSnapshotReference removeRef(String name); | ||
|
|
||
| /** | ||
| * Update branch retention what will be search by referenceName. | ||
| * | ||
| * @param ageMs For `branch` type only, a positive number for the max age of snapshots to keep in a branch while expiring snapshots, default to the value of table property `history.expire.max-snapshot-age-ms` when evaluated | ||
| * @param numToKeep For `branch` type only, a positive number for the minimum number of snapshots to keep in a branch while expiring snapshots, default to the value of table property `history.expire.min-snapshots-to-keep` when evaluated | ||
| * @param name name of snapshot reference what will be update | ||
| * @return this | ||
| */ | ||
| UpdateSnapshotReference setBranchRetention(String name, Long ageMs, Integer numToKeep); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we should use primitive values so that people must specify a value. Having a method call like But I understand we want to have the flexibility for users to use defaults. We can potentially introduce 2 different methods for these two settings separately, like
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I used code several method like setBranchSnapshotLifetime and setMinSnapshotsInBranch. But i also think its a bit overkill, if it's necessary i will add these method. |
||
|
|
||
| /** | ||
| * Update refLifetime of snapshotReference what will be search by referenceName. | ||
| * | ||
| * @param maxRefAgeMs For snapshot references except the `main` branch, default max age of snapshot references to keep while expiring snapshots. The `main` branch never expires. | ||
| * @param name name of snapshot reference what will be update | ||
| * @return this | ||
| */ | ||
| UpdateSnapshotReference setRefLifetime(String name, Long maxRefAgeMs); | ||
|
|
||
| /** | ||
| * Update name of snapshotReference what will be search by referenceName. | ||
| * | ||
| * @param oldName old name of snapshot reference | ||
| * @param name new name for snapshot reference | ||
| * @return this | ||
| */ | ||
| UpdateSnapshotReference updateName(String oldName, String name); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this one and
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. updateName is used when user want just update the name of one reference and inherit old config like reflifetime. And updateReference is used when user want update multiple properties of one old reference, so they can update these by just once operation. |
||
|
|
||
| /** | ||
| * replace old snapshotReference by new snapshotReference. | ||
| * | ||
| * @param oldName old reference name | ||
| * @param newName new reference name | ||
| * @param newReference new reference | ||
| * @return this | ||
| */ | ||
| UpdateSnapshotReference updateReference(String oldName, String newName, SnapshotReference newReference); | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Consider the situation that user A sets a retention policy for branch, and B sets retention policy for the same branch. Then both commits would succeed and one of the commit would be clobbered.
So I think the safest way to go is to not resolve commit conflicts. If the metadata is no longer current, a CommitFailedException should be thrown.