diff --git a/api/src/main/java/org/apache/iceberg/SnapshotReference.java b/api/src/main/java/org/apache/iceberg/SnapshotReference.java new file mode 100644 index 000000000000..8f305e5bb6d0 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/SnapshotReference.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import java.util.Objects; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; + +/** + * User-defined information of a named snapshot + */ +public class SnapshotReference { + + private final long snapshotId; + private final SnapshotReferenceType type; + private final Integer minSnapshotsToKeep; + private final Long maxSnapshotAgeMs; + private final Long maxRefAgeMs; + + private SnapshotReference( + long snapshotId, + SnapshotReferenceType type, + Integer minSnapshotsToKeep, + Long maxSnapshotAgeMs, + Long maxRefAgeMs) { + this.snapshotId = snapshotId; + this.type = type; + this.minSnapshotsToKeep = minSnapshotsToKeep; + this.maxSnapshotAgeMs = maxSnapshotAgeMs; + this.maxRefAgeMs = maxRefAgeMs; + } + + public long snapshotId() { + return snapshotId; + } + + public SnapshotReferenceType type() { + return type; + } + + public Integer minSnapshotsToKeep() { + return minSnapshotsToKeep; + } + + public Long maxSnapshotAgeMs() { + return maxSnapshotAgeMs; + } + + public Long maxRefAgeMs() { + return maxRefAgeMs; + } + + public static Builder builderForTag(long snapshotId) { + return builderFor(snapshotId, SnapshotReferenceType.TAG); + } + + public static Builder builderForBranch(long snapshotId) { + return builderFor(snapshotId, SnapshotReferenceType.BRANCH); + } + + public static Builder builderFrom(SnapshotReference ref) { + return new Builder(ref.type()) + .snapshotId(ref.snapshotId()) + .minSnapshotsToKeep(ref.minSnapshotsToKeep()) + .maxSnapshotAgeMs(ref.maxSnapshotAgeMs()) + .maxRefAgeMs(ref.maxRefAgeMs()); + } + + public static Builder builderFor(long snapshotId, SnapshotReferenceType type) { + return new Builder(type).snapshotId(snapshotId); + } + + public static class Builder { + + private final SnapshotReferenceType type; + + private Long snapshotId; + private Integer minSnapshotsToKeep; + private Long maxSnapshotAgeMs; + private Long maxRefAgeMs; + + Builder(SnapshotReferenceType type) { + ValidationException.check(type != null, "Snapshot reference type must not be null"); + this.type = type; + } + + public Builder snapshotId(long id) { + this.snapshotId = id; + return this; + } + + public Builder minSnapshotsToKeep(Integer value) { + this.minSnapshotsToKeep = value; + return this; + } + + public Builder maxSnapshotAgeMs(Long value) { + this.maxSnapshotAgeMs = value; + return this; + } + + public Builder maxRefAgeMs(Long value) { + this.maxRefAgeMs = value; + return this; + } + + public SnapshotReference build() { + if (type.equals(SnapshotReferenceType.TAG)) { + ValidationException.check(minSnapshotsToKeep == null, + "TAG type snapshot reference does not support setting minSnapshotsToKeep"); + ValidationException.check(maxSnapshotAgeMs == null, + "TAG type snapshot reference does not support setting maxSnapshotAgeMs"); + } else { + if (minSnapshotsToKeep != null) { + ValidationException.check(minSnapshotsToKeep > 0, + "Min snapshots to keep must be greater than 0"); + } + + if (maxSnapshotAgeMs != null) { + ValidationException.check(maxSnapshotAgeMs > 0, "Max snapshot age must be greater than 0"); + } + } + + if (maxRefAgeMs != null) { + ValidationException.check(maxRefAgeMs > 0, "Max reference age must be greater than 0"); + } + + return new SnapshotReference(snapshotId, type, minSnapshotsToKeep, maxSnapshotAgeMs, maxRefAgeMs); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o == null || getClass() != o.getClass()) { + return false; + } + + SnapshotReference that = (SnapshotReference) o; + return snapshotId == that.snapshotId && + type == that.type && + Objects.equals(minSnapshotsToKeep, that.minSnapshotsToKeep) && + Objects.equals(maxSnapshotAgeMs, that.maxSnapshotAgeMs) && + Objects.equals(maxRefAgeMs, that.maxRefAgeMs); + } + + @Override + public int hashCode() { + return Objects.hash(snapshotId, type, minSnapshotsToKeep, maxSnapshotAgeMs, maxRefAgeMs); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("snapshotId", snapshotId) + .add("type", type) + .add("minSnapshotsToKeep", minSnapshotsToKeep) + .add("maxSnapshotAgeMs", maxSnapshotAgeMs) + .add("maxRefAgeMs", maxRefAgeMs) + .toString(); + } +} diff --git a/api/src/main/java/org/apache/iceberg/SnapshotReferenceType.java b/api/src/main/java/org/apache/iceberg/SnapshotReferenceType.java new file mode 100644 index 000000000000..ee6440d2439f --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/SnapshotReferenceType.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +enum SnapshotReferenceType { + BRANCH, + TAG +} diff --git a/api/src/main/java/org/apache/iceberg/UpdateSnapshotReference.java b/api/src/main/java/org/apache/iceberg/UpdateSnapshotReference.java new file mode 100644 index 000000000000..c824f1b068bc --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/UpdateSnapshotReference.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import java.util.Map; +import org.apache.iceberg.exceptions.CommitFailedException; + +/** + * API for updating snapshot reference. + *

+ * When committing, these changes will be applied to the current table metadata. Commit conflicts + * will not be resolved and will result in a {@link CommitFailedException}. + */ +public interface UpdateSnapshotReference extends PendingUpdate> { + + /** + * remove snapshotReference. + * + * @param name name of snapshot reference + * @return this + * @throws IllegalArgumentException If there is no such snapshot reference named name + */ + UpdateSnapshotReference removeRef(String name); + + /** + * Update branch retention what will be search by referenceName. + * + * @param ageMs For `branch` type only, a positive number for the max age of snapshots to keep in a branch while expiring snapshots, default to the value of table property `history.expire.max-snapshot-age-ms` when evaluated + * @param numToKeep For `branch` type only, a positive number for the minimum number of snapshots to keep in a branch while expiring snapshots, default to the value of table property `history.expire.min-snapshots-to-keep` when evaluated + * @param name name of snapshot reference what will be update + * @return this + */ + UpdateSnapshotReference setBranchRetention(String name, Long ageMs, Integer numToKeep); + + /** + * Update refLifetime of snapshotReference what will be search by referenceName. + * + * @param maxRefAgeMs For snapshot references except the `main` branch, default max age of snapshot references to keep while expiring snapshots. The `main` branch never expires. + * @param name name of snapshot reference what will be update + * @return this + */ + UpdateSnapshotReference setRefLifetime(String name, Long maxRefAgeMs); + + /** + * Update name of snapshotReference what will be search by referenceName. + * + * @param oldName old name of snapshot reference + * @param name new name for snapshot reference + * @return this + */ + UpdateSnapshotReference updateName(String oldName, String name); + + /** + * replace old snapshotReference by new snapshotReference. + * + * @param oldName old reference name + * @param newName new reference name + * @param newReference new reference + * @return this + */ + UpdateSnapshotReference updateReference(String oldName, String newName, SnapshotReference newReference); +} diff --git a/api/src/test/java/org/apache/iceberg/TestSnapshotReference.java b/api/src/test/java/org/apache/iceberg/TestSnapshotReference.java new file mode 100644 index 000000000000..60653f6a0421 --- /dev/null +++ b/api/src/test/java/org/apache/iceberg/TestSnapshotReference.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import org.apache.iceberg.exceptions.ValidationException; +import org.junit.Assert; +import org.junit.Test; + +public class TestSnapshotReference { + + @Test + public void testTagDefault() { + SnapshotReference ref = SnapshotReference.builderForTag(1L).build(); + Assert.assertEquals(1L, ref.snapshotId()); + Assert.assertEquals(SnapshotReferenceType.TAG, ref.type()); + Assert.assertNull(ref.minSnapshotsToKeep()); + Assert.assertNull(ref.maxSnapshotAgeMs()); + Assert.assertNull(ref.maxRefAgeMs()); + } + + @Test + public void testBranchDefault() { + SnapshotReference ref = SnapshotReference.builderForBranch(1L).build(); + Assert.assertEquals(1L, ref.snapshotId()); + Assert.assertEquals(SnapshotReferenceType.BRANCH, ref.type()); + Assert.assertNull(ref.minSnapshotsToKeep()); + Assert.assertNull(ref.maxSnapshotAgeMs()); + } + + @Test + public void testTagWithOverride() { + SnapshotReference ref = SnapshotReference.builderForBranch(1L) + .maxRefAgeMs(10L) + .build(); + Assert.assertEquals(1L, ref.snapshotId()); + Assert.assertEquals(SnapshotReferenceType.BRANCH, ref.type()); + Assert.assertEquals(10L, (long) ref.maxRefAgeMs()); + } + + @Test + public void testBranchWithOverride() { + SnapshotReference ref = SnapshotReference.builderForBranch(1L) + .minSnapshotsToKeep(10) + .maxSnapshotAgeMs(20L) + .maxRefAgeMs(30L) + .build(); + Assert.assertEquals(1L, ref.snapshotId()); + Assert.assertEquals(SnapshotReferenceType.BRANCH, ref.type()); + Assert.assertEquals(10, (int) ref.minSnapshotsToKeep()); + Assert.assertEquals(20L, (long) ref.maxSnapshotAgeMs()); + Assert.assertEquals(30L, (long) ref.maxRefAgeMs()); + } + + @Test + public void testNoTypeFailure() { + AssertHelpers.assertThrows("Snapshot reference type must be specified", + ValidationException.class, + "Snapshot reference type must not be null", + () -> SnapshotReference.builderFor(1L, null).build()); + } + + @Test + public void testTagBuildFailures() { + AssertHelpers.assertThrows("Max reference age must be greater than 0 for tag", + ValidationException.class, + "Max reference age must be greater than 0", + () -> SnapshotReference.builderForTag(1L) + .maxRefAgeMs(-1L) + .build()); + + AssertHelpers.assertThrows("TAG type snapshot reference does not support setting minSnapshotsToKeep", + ValidationException.class, + "TAG type snapshot reference does not support setting minSnapshotsToKeep", + () -> SnapshotReference.builderForTag(1L) + .minSnapshotsToKeep(2) + .build()); + + AssertHelpers.assertThrows("TAG type snapshot reference does not support setting maxSnapshotAgeMs", + ValidationException.class, + "TAG type snapshot reference does not support setting maxSnapshotAgeMs", + () -> SnapshotReference.builderForTag(1L) + .maxSnapshotAgeMs(2L) + .build()); + } + + @Test + public void testBranchBuildFailures() { + AssertHelpers.assertThrows("Max snapshot age must be greater than 0", + ValidationException.class, + "Max snapshot age must be greater than 0", + () -> SnapshotReference.builderForBranch(1L) + .maxSnapshotAgeMs(-1L) + .build()); + + AssertHelpers.assertThrows("Min snapshots to keep must be greater than 0", + ValidationException.class, + "Min snapshots to keep must be greater than 0", + () -> SnapshotReference.builderForBranch(1L) + .minSnapshotsToKeep(-1) + .build()); + + AssertHelpers.assertThrows("Max reference age must be greater than 0 for branch", + ValidationException.class, + "Max reference age must be greater than 0", + () -> SnapshotReference.builderForBranch(1L) + .maxRefAgeMs(-1L) + .build()); + } +} diff --git a/core/src/main/java/org/apache/iceberg/SnapshotReferenceParser.java b/core/src/main/java/org/apache/iceberg/SnapshotReferenceParser.java new file mode 100644 index 000000000000..11f37af5eea4 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/SnapshotReferenceParser.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.io.StringWriter; +import java.io.UncheckedIOException; +import java.util.Locale; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.util.JsonUtil; + +public class SnapshotReferenceParser { + + private SnapshotReferenceParser() { + } + + private static final String SNAPSHOT_ID = "snapshot-id"; + private static final String TYPE = "type"; + private static final String MIN_SNAPSHOTS_TO_KEEP = "min-snapshots-to-keep"; + private static final String MAX_SNAPSHOT_AGE_MS = "max-snapshot-age-ms"; + private static final String MAX_REF_AGE_MS = "max-ref-age-ms"; + + public static String toJson(SnapshotReference ref) { + return toJson(ref, false); + } + + public static String toJson(SnapshotReference ref, boolean pretty) { + try { + StringWriter writer = new StringWriter(); + JsonGenerator generator = JsonUtil.factory().createGenerator(writer); + if (pretty) { + generator.useDefaultPrettyPrinter(); + } + + toJson(ref, generator); + generator.flush(); + return writer.toString(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static void toJson(SnapshotReference ref, JsonGenerator generator) throws IOException { + generator.writeStartObject(); + generator.writeNumberField(SNAPSHOT_ID, ref.snapshotId()); + generator.writeStringField(TYPE, ref.type().name().toLowerCase(Locale.ENGLISH)); + JsonUtil.writeIntegerIfExists(MIN_SNAPSHOTS_TO_KEEP, ref.minSnapshotsToKeep(), generator); + JsonUtil.writeLongIfExists(MAX_SNAPSHOT_AGE_MS, ref.maxSnapshotAgeMs(), generator); + JsonUtil.writeLongIfExists(MAX_REF_AGE_MS, ref.maxRefAgeMs(), generator); + generator.writeEndObject(); + } + + public static SnapshotReference fromJson(String json) { + try { + return fromJson(JsonUtil.mapper().readValue(json, JsonNode.class)); + } catch (IOException e) { + throw new UncheckedIOException("Failed to parse snapshot ref: " + json, e); + } + } + + public static SnapshotReference fromJson(JsonNode node) { + Preconditions.checkArgument(node.isObject(), "Cannot parse snapshot reference from a non-object: %s", node); + long snapshotId = JsonUtil.getLong(SNAPSHOT_ID, node); + SnapshotReferenceType type = SnapshotReferenceType.valueOf( + JsonUtil.getString(TYPE, node).toUpperCase(Locale.ENGLISH)); + Integer minSnapshotsToKeep = JsonUtil.getIntOrNull(MIN_SNAPSHOTS_TO_KEEP, node); + Long maxSnapshotAgeMs = JsonUtil.getLongOrNull(MAX_SNAPSHOT_AGE_MS, node); + Long maxRefAgeMs = JsonUtil.getLongOrNull(MAX_REF_AGE_MS, node); + return SnapshotReference.builderFor(snapshotId, type) + .minSnapshotsToKeep(minSnapshotsToKeep) + .maxSnapshotAgeMs(maxSnapshotAgeMs) + .maxRefAgeMs(maxRefAgeMs) + .build(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/SnapshotReferenceUpdate.java b/core/src/main/java/org/apache/iceberg/SnapshotReferenceUpdate.java new file mode 100644 index 000000000000..a2e1386fbf59 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/SnapshotReferenceUpdate.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; + +public class SnapshotReferenceUpdate implements UpdateSnapshotReference { + + private final TableOperations ops; + private TableMetadata base; + private final Map update = Maps.newHashMap(); + private final Set removals = Sets.newHashSet(); + + SnapshotReferenceUpdate(TableOperations ops) { + this.ops = ops; + this.base = ops.current(); + } + + @Override + public Map apply() { + this.base = ops.refresh(); + + Map newRefs = Maps.newHashMap(); + for (String s : base.refs().keySet()) { + if (!removals.contains(s)) { + newRefs.put(s, base.refs().get(s)); + } + } + + newRefs.putAll(update); + + return newRefs; + } + + @Override + public void commit() { + ops.commit(base, base.updateSnapshotReference(apply())); + } + + @Override + public UpdateSnapshotReference removeRef(String name) { + ValidationException.check(name != null, "Snapshot reference name can't be null"); + ValidationException.check(base.refs().containsKey(name), "Can't find snapshot reference named %s", name); + ValidationException.check(!removals.contains(name), "Update multiple properties for snapshot reference should use" + + " method updateReference(SnapshotReference oldReference, SnapshotReference newReference)"); + + removals.add(name); + + return this; + } + + @Override + public UpdateSnapshotReference setBranchRetention(String name, Long ageMs, Integer numToKeep) { + ValidationException.check(name != null, "Snapshot reference name can't be null"); + SnapshotReference baseReference = base.refs().get(name); + ValidationException.check(baseReference != null, "Can't find snapshot reference named %s", name); + + SnapshotReference.Builder builder = SnapshotReference.builderFor( + baseReference.snapshotId(), + baseReference.type()); + + Long newAgeMs = ageMs == null ? baseReference.maxSnapshotAgeMs() : ageMs; + Integer newNumToKeep = numToKeep == null ? baseReference.minSnapshotsToKeep() : numToKeep; + + builder.maxSnapshotAgeMs(newAgeMs); + builder.minSnapshotsToKeep(newNumToKeep); + + SnapshotReference newRef = builder.build(); + + update.put(name, newRef); + removals.add(name); + + return this; + } + + @Override + public UpdateSnapshotReference setRefLifetime(String name, Long maxRefAgeMs) { + ValidationException.check(name != null, "Snapshot reference name can't be null"); + SnapshotReference baseReference = base.refs().get(name); + ValidationException.check(baseReference != null, "Can't find snapshot reference named %s", name); + + SnapshotReference.Builder builder = SnapshotReference.builderFor( + baseReference.snapshotId(), + baseReference.type()); + + Long newMaxRefAgeMs = maxRefAgeMs == null ? baseReference.maxRefAgeMs() : maxRefAgeMs; + + builder.maxRefAgeMs(newMaxRefAgeMs); + + SnapshotReference newRef = builder.build(); + + update.put(name, newRef); + removals.add(name); + + return this; + } + + @Override + public UpdateSnapshotReference updateName(String oldName, String name) { + ValidationException.check(oldName != null, "Old snapshot reference name can't be null"); + ValidationException.check(name != null, "new snapshot reference name can't be null"); + SnapshotReference baseReference = base.refs().get(oldName); + ValidationException.check(baseReference != null, "Can't find snapshot reference named %s", oldName); + ValidationException.check( + !removals.contains(oldName), + "Update multiple properties for snapshot reference should use" + + " method updateReference(SnapshotReference oldReference, SnapshotReference newReference)"); + + SnapshotReference newRef = SnapshotReference.builderFor(baseReference.snapshotId(), baseReference.type()) + .maxSnapshotAgeMs(baseReference.maxSnapshotAgeMs()) + .minSnapshotsToKeep(baseReference.minSnapshotsToKeep()) + .build(); + + update.put(name, newRef); + removals.add(oldName); + + return this; + } + + @Override + public UpdateSnapshotReference updateReference( + String oldName, String newName, SnapshotReference newReference) { + ValidationException.check(oldName != null, "Old snapshot reference name can't be null"); + ValidationException.check(newName != null, "New snapshot reference name can't be null"); + ValidationException.check( + !removals.contains(oldName), + "Update multiple properties for snapshot reference should use" + + " method updateReference(SnapshotReference oldReference, SnapshotReference newReference)"); + ValidationException.check( + base.refs().containsKey(oldName), + "There is no such snapshot reference named %s", oldName); + ValidationException.check(base.refs().get(oldName).type().equals(newReference.type()), "The type of snapshot " + + "reference can't change"); + + removals.add(oldName); + update.put(newName, newReference); + + return this; + } +} diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 12e4fc2e1e07..3c8c562947f5 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -36,9 +36,11 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSetMultimap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.SetMultimap; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.TypeUtil; @@ -56,6 +58,7 @@ public class TableMetadata implements Serializable { static final int INITIAL_SPEC_ID = 0; static final int INITIAL_SORT_ORDER_ID = 1; static final int INITIAL_SCHEMA_ID = 0; + static final String MAIN_BRANCH = "main"; private static final long ONE_MINUTE = TimeUnit.MINUTES.toMillis(1); @@ -126,7 +129,7 @@ static TableMetadata newTableMetadata(Schema schema, freshSpec.specId(), ImmutableList.of(freshSpec), freshSpec.lastAssignedFieldId(), freshSortOrderId, ImmutableList.of(freshSortOrder), ImmutableMap.copyOf(properties), -1, ImmutableList.of(), - ImmutableList.of(), ImmutableList.of()); + ImmutableList.of(), ImmutableList.of(), ImmutableMap.of()); } public static class SnapshotLogEntry implements HistoryEntry { @@ -240,6 +243,8 @@ public String toString() { private final Map sortOrdersById; private final List snapshotLog; private final List previousFiles; + private final Map refs; + private final SetMultimap refsById; @SuppressWarnings("checkstyle:CyclomaticComplexity") TableMetadata(String metadataFileLocation, @@ -260,7 +265,8 @@ public String toString() { long currentSnapshotId, List snapshots, List snapshotLog, - List previousFiles) { + List previousFiles, + Map refs) { Preconditions.checkArgument(specs != null && !specs.isEmpty(), "Partition specs cannot be null or empty"); Preconditions.checkArgument(sortOrders != null && !sortOrders.isEmpty(), "Sort orders cannot be null or empty"); Preconditions.checkArgument(formatVersion <= SUPPORTED_TABLE_FORMAT_VERSION, @@ -294,6 +300,8 @@ public String toString() { this.schemasById = indexSchemas(); this.specsById = indexSpecs(specs); this.sortOrdersById = indexSortOrders(sortOrders); + this.refs = validateAndCompleteRefs(refs); + this.refsById = indexRefs(); HistoryEntry last = null; for (HistoryEntry logEntry : snapshotLog) { @@ -467,14 +475,22 @@ public List previousFiles() { return previousFiles; } + public Map refs() { + return refs; + } + + public SetMultimap refsById() { + return refsById; + } + public TableMetadata withUUID() { if (uuid != null) { return this; } else { return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, lastSequenceNumber, lastUpdatedMillis, lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, - currentSnapshotId, snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, + snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); } } @@ -500,7 +516,7 @@ public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) { lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, newSchemaId, builder.build(), defaultSpecId, updatedSpecs, lastAssignedPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); } // The caller is responsible to pass a newPartitionSpec with correct partition field IDs @@ -537,8 +553,8 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, newDefaultSpecId, builder.build(), Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId()), - defaultSortOrderId, sortOrders, properties, - currentSnapshotId, snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, + addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); } public TableMetadata replaceSortOrder(SortOrder newOrder) { @@ -576,7 +592,7 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); } public TableMetadata addStagedSnapshot(Snapshot snapshot) { @@ -593,7 +609,7 @@ public TableMetadata addStagedSnapshot(Snapshot snapshot) { snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, newSnapshots, snapshotLog, - addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); } public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { @@ -615,18 +631,26 @@ public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { .add(new SnapshotLogEntry(snapshot.timestampMillis(), snapshot.snapshotId())) .build(); + SnapshotReference mainBranch = SnapshotReference.builderFrom(refs.get(MAIN_BRANCH)) + .snapshotId(snapshot.snapshotId()) + .build(); + Map newRefs = new ImmutableMap.Builder() + .putAll(refs()) + .put(MAIN_BRANCH, mainBranch) + .build(); + return new TableMetadata(null, formatVersion, uuid, location, snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), newSnapshots, newSnapshotLog, - addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + addPreviousFile(metadataFileLocation, lastUpdatedMillis), newRefs); } public TableMetadata removeSnapshotsIf(Predicate removeIf) { List filtered = Lists.newArrayListWithExpectedSize(snapshots.size()); for (Snapshot snapshot : snapshots) { - // keep the current snapshot and any snapshots that do not match the removeIf condition - if (snapshot.snapshotId() == currentSnapshotId || !removeIf.test(snapshot)) { + // keep snapshots with reference and any snapshots that do not match the removeIf condition + if (refsById.containsKey(snapshot.snapshotId()) || !removeIf.test(snapshot)) { filtered.add(snapshot); } } @@ -651,7 +675,7 @@ public TableMetadata removeSnapshotsIf(Predicate removeIf) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, - ImmutableList.copyOf(newSnapshotLog), addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + ImmutableList.copyOf(newSnapshotLog), addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); } private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { @@ -672,10 +696,18 @@ private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { .add(new SnapshotLogEntry(nowMillis, snapshot.snapshotId())) .build(); + SnapshotReference mainBranch = SnapshotReference.builderFrom(refs.get(MAIN_BRANCH)) + .snapshotId(snapshot.snapshotId()) + .build(); + Map newRefs = new ImmutableMap.Builder() + .putAll(refs()) + .put(MAIN_BRANCH, mainBranch) + .build(); + return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, nowMillis, lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, - newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis), newRefs); } public TableMetadata replaceProperties(Map rawProperties) { @@ -684,7 +716,7 @@ public TableMetadata replaceProperties(Map rawProperties) { TableMetadata metadata = new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, - snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis, newProperties)); + snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis, newProperties), refs); int newFormatVersion = PropertyUtil.propertyAsInt(rawProperties, TableProperties.FORMAT_VERSION, formatVersion); if (formatVersion != newFormatVersion) { @@ -710,7 +742,14 @@ public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + snapshots, newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); + } + + public TableMetadata updateSnapshotReference(Map newRefs) { + return new TableMetadata(null, formatVersion, uuid, location, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, + snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis), newRefs); } private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil.NextID nextID) { @@ -834,11 +873,20 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update schemasBuilder.add(new Schema(freshSchemaId, freshSchema.columns(), freshSchema.identifierFieldIds())); } + SnapshotReference mainBranch = SnapshotReference.builderFrom(refs.get(MAIN_BRANCH)) + .snapshotId(-1) + .build(); + Map newRefs = new ImmutableMap.Builder() + .putAll(refs()) + .put(MAIN_BRANCH, mainBranch) + .build(); + TableMetadata metadata = new TableMetadata(null, formatVersion, uuid, newLocation, lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchemaId, schemasBuilder.build(), specId, specListBuilder.build(), Math.max(lastAssignedPartitionId, newSpec.lastAssignedFieldId()), orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties), - -1, snapshots, ImmutableList.of(), addPreviousFile(metadataFileLocation, lastUpdatedMillis, newProperties)); + -1, snapshots, ImmutableList.of(), + addPreviousFile(metadataFileLocation, lastUpdatedMillis, newProperties), newRefs); if (formatVersion != newFormatVersion) { metadata = metadata.upgradeToFormatVersion(newFormatVersion); @@ -851,7 +899,7 @@ public TableMetadata updateLocation(String newLocation) { return new TableMetadata(null, formatVersion, uuid, newLocation, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); } public TableMetadata upgradeToFormatVersion(int newFormatVersion) { @@ -868,7 +916,7 @@ public TableMetadata upgradeToFormatVersion(int newFormatVersion) { return new TableMetadata(null, newFormatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis), refs); } private List addPreviousFile(String previousFileLocation, long timestampMillis) { @@ -989,6 +1037,31 @@ private static Map indexSortOrders(List sortOrder return builder.build(); } + private Map validateAndCompleteRefs(Map inputRefs) { + for (SnapshotReference ref : inputRefs.values()) { + Preconditions.checkArgument(snapshotsById.containsKey(ref.snapshotId()), + "Snapshot reference %s does not exist in the existing snapshots list", ref); + } + + if (!inputRefs.containsKey(MAIN_BRANCH)) { + return new ImmutableMap.Builder() + .putAll(inputRefs) + .put(MAIN_BRANCH, SnapshotReference.builderForBranch(currentSnapshotId).build()) + .build(); + } + + return inputRefs; + } + + private SetMultimap indexRefs() { + ImmutableSetMultimap.Builder builder = ImmutableSetMultimap.builder(); + for (SnapshotReference ref : refs.values()) { + builder.put(ref.snapshotId(), ref); + } + + return builder.build(); + } + private int reuseOrCreateNewSchemaId(Schema newSchema) { // if the schema already exists, use its id; otherwise use the highest id + 1 int newSchemaId = currentSchemaId; diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index 1e26fbb9b11b..a6ad04e43eab 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -42,9 +42,11 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.util.JsonUtil; +@SuppressWarnings({"checkstyle:CyclomaticComplexity", "checkstyle:MethodLength"}) public class TableMetadataParser { public enum Codec { @@ -105,6 +107,7 @@ private TableMetadataParser() { static final String SNAPSHOT_LOG = "snapshot-log"; static final String METADATA_FILE = "metadata-file"; static final String METADATA_LOG = "metadata-log"; + static final String REFS = "refs"; public static void overwrite(TableMetadata metadata, OutputFile outputFile) { internalWrite(metadata, outputFile, true); @@ -238,6 +241,13 @@ private static void toJson(TableMetadata metadata, JsonGenerator generator) thro } generator.writeEndArray(); + generator.writeObjectFieldStart(REFS); + for (Map.Entry ref : metadata.refs().entrySet()) { + generator.writeFieldName(ref.getKey()); + SnapshotReferenceParser.toJson(ref.getValue(), generator); + } + generator.writeEndObject(); + generator.writeEndObject(); } @@ -430,9 +440,18 @@ static TableMetadata fromJson(FileIO io, String metadataLocation, JsonNode node) } } + ImmutableMap.Builder refs = ImmutableMap.builder(); + if (node.has(REFS)) { + Iterator> refIterator = node.get(REFS).fields(); + while (refIterator.hasNext()) { + Map.Entry refEntry = refIterator.next(); + refs.put(refEntry.getKey(), SnapshotReferenceParser.fromJson(refEntry.getValue())); + } + } + return new TableMetadata(metadataLocation, formatVersion, uuid, location, lastSequenceNumber, lastUpdatedMillis, lastAssignedColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentVersionId, - snapshots, entries.build(), metadataEntries.build()); + snapshots, entries.build(), metadataEntries.build(), refs.build()); } } diff --git a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java index bcde95968484..fac90523b461 100644 --- a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java @@ -20,8 +20,10 @@ package org.apache.iceberg.util; import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -73,6 +75,18 @@ public static long getLong(String property, JsonNode node) { return pNode.asLong(); } + public static Long getLongOrNull(String property, JsonNode node) { + if (!node.has(property)) { + return null; + } + + Preconditions.checkArgument(node.has(property), "Cannot parse missing long %s", property); + JsonNode pNode = node.get(property); + Preconditions.checkArgument(pNode != null && !pNode.isNull() && pNode.isIntegralNumber() && + pNode.canConvertToLong(), "Cannot parse %s from non-numeric value: %s", property, pNode); + return pNode.asLong(); + } + public static boolean getBool(String property, JsonNode node) { Preconditions.checkArgument(node.has(property), "Cannot parse missing boolean %s", property); JsonNode pNode = node.get(property); @@ -134,6 +148,18 @@ public static Set getIntegerSetOrNull(String property, JsonNode node) { .build(); } + public static void writeIntegerIfExists(String key, Integer value, JsonGenerator generator) throws IOException { + if (value != null) { + generator.writeNumberField(key, value); + } + } + + public static void writeLongIfExists(String key, Long value, JsonGenerator generator) throws IOException { + if (value != null) { + generator.writeNumberField(key, value); + } + } + abstract static class JsonArrayIterator implements Iterator { private final Iterator elements; diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotReferenceParser.java b/core/src/test/java/org/apache/iceberg/TestSnapshotReferenceParser.java new file mode 100644 index 000000000000..9c1f72ff6f1a --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotReferenceParser.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import org.junit.Assert; +import org.junit.Test; + +public class TestSnapshotReferenceParser { + + @Test + public void testTagToJsonDefault() { + String json = "{\"snapshot-id\":1,\"type\":\"tag\"}"; + SnapshotReference ref = SnapshotReference.builderForTag(1L).build(); + Assert.assertEquals("Should be able to serialize default tag", + json, SnapshotReferenceParser.toJson(ref)); + } + + @Test + public void testTagToJsonAllFields() { + String json = "{\"snapshot-id\":1,\"type\":\"tag\",\"max-ref-age-ms\":1}"; + SnapshotReference ref = SnapshotReference.builderForTag(1L) + .maxRefAgeMs(1L) + .build(); + Assert.assertEquals("Should be able to serialize tag with all fields", + json, SnapshotReferenceParser.toJson(ref)); + } + + @Test + public void testBranchToJsonDefault() { + String json = "{\"snapshot-id\":1,\"type\":\"branch\"}"; + SnapshotReference ref = SnapshotReference.builderForBranch(1L).build(); + Assert.assertEquals("Should be able to serialize default branch", + json, SnapshotReferenceParser.toJson(ref)); + } + + @Test + public void testBranchToJsonAllFields() { + String json = "{\"snapshot-id\":1,\"type\":\"branch\",\"min-snapshots-to-keep\":2," + + "\"max-snapshot-age-ms\":3,\"max-ref-age-ms\":4}"; + SnapshotReference ref = SnapshotReference.builderForBranch(1L) + .minSnapshotsToKeep(2) + .maxSnapshotAgeMs(3L) + .maxRefAgeMs(4L) + .build(); + Assert.assertEquals("Should be able to serialize branch with all fields", + json, SnapshotReferenceParser.toJson(ref)); + } + + @Test + public void testTagFromJsonDefault() { + String json = "{\"snapshot-id\":1,\"type\":\"tag\"}"; + SnapshotReference ref = SnapshotReference.builderForTag(1L).build(); + Assert.assertEquals("Should be able to deserialize default tag", + ref, SnapshotReferenceParser.fromJson(json)); + } + + @Test + public void testTagFromJsonAllFields() { + String json = "{\"snapshot-id\":1,\"type\":\"tag\",\"max-ref-age-ms\":1}"; + SnapshotReference ref = SnapshotReference.builderForTag(1L) + .maxRefAgeMs(1L) + .build(); + Assert.assertEquals("Should be able to deserialize tag with all fields", + ref, SnapshotReferenceParser.fromJson(json)); + } + + @Test + public void testBranchFromJsonDefault() { + String json = "{\"snapshot-id\":1,\"type\":\"branch\"}"; + SnapshotReference ref = SnapshotReference.builderForBranch(1L).build(); + Assert.assertEquals("Should be able to deserialize default branch", + ref, SnapshotReferenceParser.fromJson(json)); + } + + @Test + public void testBranchFromJsonAllFields() { + String json = "{\"snapshot-id\":1,\"type\":\"branch\",\"min-snapshots-to-keep\":2," + + "\"max-snapshot-age-ms\":3,\"max-ref-age-ms\":4}"; + SnapshotReference ref = SnapshotReference.builderForBranch(1L) + .minSnapshotsToKeep(2) + .maxSnapshotAgeMs(3L) + .maxRefAgeMs(4L) + .build(); + Assert.assertEquals("Should be able to deserialize branch with all fields", + ref, SnapshotReferenceParser.fromJson(json)); + } + +} diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotReferenceUpdate.java b/core/src/test/java/org/apache/iceberg/TestSnapshotReferenceUpdate.java new file mode 100644 index 000000000000..ecb87c662dbc --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotReferenceUpdate.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import java.io.File; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestSnapshotReferenceUpdate extends TableTestBase { + + private static final int FORMAT_V2 = 2; + + public TestSnapshotReferenceUpdate() { + super(FORMAT_V2); + } + + @Before + public void setupTable() throws Exception { + this.tableDir = temp.newFolder(); + Assert.assertTrue(tableDir.delete()); // created during table creation + + this.metadataDir = new File(tableDir, "metadata"); + this.table = create(SCHEMA, PartitionSpec.unpartitioned()); + SnapshotReference newBranchRef = SnapshotReference.builderFor( + 123, + SnapshotReferenceType.BRANCH) + .maxSnapshotAgeMs(1L) + .minSnapshotsToKeep(1) + .build(); + SnapshotReference newTagRef = SnapshotReference.builderFor( + 123, + SnapshotReferenceType.TAG) + .maxRefAgeMs(1L) + .build(); + TableMetadata base = table.ops().current(); + Map refs = Maps.newHashMap(); + refs.put("testTag", newTagRef); + refs.put("testBranch", newBranchRef); + List branchSnapshot = + Lists.newArrayList(new BaseSnapshot(table.ops().io(), 123, null, "file:/tmp/manifest1" + + ".avro")); + TableMetadata newTableMetadata = new TableMetadata(metadataDir.getAbsolutePath(), base.formatVersion(), base.uuid(), + base.location(), + base.lastSequenceNumber(), base.lastUpdatedMillis(), base.lastColumnId(), base.currentSchemaId(), + base.schemas(), base.defaultSpecId(), base.specs(), base.lastAssignedPartitionId(), + base.defaultSortOrderId(), base.sortOrders(), base.properties(), 123, branchSnapshot, + base.snapshotLog(), base.previousFiles(), refs); + table.ops().commit(base, newTableMetadata); + } + + @Test + public void testRemoveReference() { + SnapshotReferenceUpdate updateSnapshotReference = new SnapshotReferenceUpdate(table.ops()); + updateSnapshotReference.removeRef("testBranch"); + updateSnapshotReference.commit(); + table.refresh(); + Assert.assertNull(table.ops().current().refs().get("testBranch")); + } + + @Test + public void testSetBranchRetention() { + SnapshotReferenceUpdate updateSnapshotReference = new SnapshotReferenceUpdate(table.ops()); + updateSnapshotReference.setBranchRetention("testBranch", null, 2); + updateSnapshotReference.commit(); + table.refresh(); + Assert.assertEquals(table.ops().current().refs().get("testBranch").maxSnapshotAgeMs().longValue(), 1); + Assert.assertEquals(table.ops().current().refs().get("testBranch").minSnapshotsToKeep().longValue(), 2); + } + + @Test + public void testSetMaxRefAgeMs() { + SnapshotReferenceUpdate updateSnapshotReference = new SnapshotReferenceUpdate(table.ops()); + updateSnapshotReference.setRefLifetime("testBranch", 2L); + updateSnapshotReference.commit(); + table.refresh(); + Assert.assertEquals(table.ops().current().refs().get("testBranch").maxRefAgeMs().longValue(), 2); + } + + @Test + public void testUpName() { + SnapshotReferenceUpdate updateSnapshotReference = new SnapshotReferenceUpdate(table.ops()); + updateSnapshotReference.updateName("testBranch", "newTestBranch"); + updateSnapshotReference.commit(); + table.refresh(); + Assert.assertFalse(table.ops().current().refs().containsKey("testBranch")); + Assert.assertTrue(table.ops().current().refs().containsKey("newTestBranch")); + } + + @Test + public void testUpdateReference() { + SnapshotReference newTagRef = SnapshotReference.builderFor( + 123, + SnapshotReferenceType.BRANCH) + .maxSnapshotAgeMs(3L) + .build(); + SnapshotReferenceUpdate updateSnapshotReference = new SnapshotReferenceUpdate(table.ops()); + updateSnapshotReference.updateReference("testBranch", "newTestBranch", newTagRef); + updateSnapshotReference.commit(); + table.refresh(); + Assert.assertFalse(table.ops().current().refs().containsKey("testBranch")); + Assert.assertTrue(table.ops().current().refs().containsKey("newTestBranch")); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 6ddbeab1d1d4..409b7a0c5c48 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -108,7 +108,8 @@ public void testJsonConversion() throws Exception { 7, ImmutableList.of(TEST_SCHEMA, schema), 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, - Arrays.asList(previousSnapshot, currentSnapshot), snapshotLog, ImmutableList.of()); + Arrays.asList(previousSnapshot, currentSnapshot), snapshotLog, ImmutableList.of(), + ImmutableMap.of(TableMetadata.MAIN_BRANCH, SnapshotReference.builderForBranch(currentSnapshotId).build())); String asJson = TableMetadataParser.toJson(expected); TableMetadata metadata = TableMetadataParser.fromJson(ops.io(), asJson); @@ -159,6 +160,8 @@ public void testJsonConversion() throws Exception { metadata.snapshot(previousSnapshotId).allManifests()); Assert.assertNull("Previous snapshot's schema ID should be null", metadata.snapshot(previousSnapshotId).schemaId()); + Assert.assertEquals("Refs should match", + expected.refs(), metadata.refs()); } @Test @@ -180,7 +183,8 @@ public void testBackwardCompat() throws Exception { 0, System.currentTimeMillis(), 3, TableMetadata.INITIAL_SCHEMA_ID, ImmutableList.of(schema), 6, ImmutableList.of(spec), spec.lastAssignedFieldId(), TableMetadata.INITIAL_SORT_ORDER_ID, ImmutableList.of(sortOrder), ImmutableMap.of("property", "value"), - currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), ImmutableList.of(), ImmutableList.of()); + currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), ImmutableList.of(), ImmutableList.of(), + ImmutableMap.of(TableMetadata.MAIN_BRANCH, SnapshotReference.builderForBranch(currentSnapshotId).build())); String asJson = toJsonWithoutSpecAndSchemaList(expected); TableMetadata metadata = TableMetadataParser.fromJson(ops.io(), asJson); @@ -233,6 +237,8 @@ public void testBackwardCompat() throws Exception { expected.previousFiles(), metadata.previousFiles()); Assert.assertNull("Previous snapshot's schema ID should be null", metadata.snapshot(previousSnapshotId).schemaId()); + Assert.assertEquals("Refs should match", + expected.refs(), metadata.refs()); } private static String toJsonWithoutSpecAndSchemaList(TableMetadata metadata) { @@ -302,7 +308,8 @@ public void testJsonWithPreviousMetadataLog() throws Exception { 7, ImmutableList.of(TEST_SCHEMA), 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, - ImmutableList.copyOf(previousMetadataLog)); + ImmutableList.copyOf(previousMetadataLog), + ImmutableMap.of(TableMetadata.MAIN_BRANCH, SnapshotReference.builderForBranch(currentSnapshotId).build())); String asJson = TableMetadataParser.toJson(base); TableMetadata metadataFromJson = TableMetadataParser.fromJson(ops.io(), asJson); @@ -337,7 +344,8 @@ public void testAddPreviousMetadataRemoveNone() { 7, ImmutableList.of(TEST_SCHEMA), 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, - ImmutableList.copyOf(previousMetadataLog)); + ImmutableList.copyOf(previousMetadataLog), + ImmutableMap.of(TableMetadata.MAIN_BRANCH, SnapshotReference.builderForBranch(currentSnapshotId).build())); previousMetadataLog.add(latestPreviousMetadata); @@ -384,7 +392,8 @@ public void testAddPreviousMetadataRemoveOne() { ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, - ImmutableList.copyOf(previousMetadataLog)); + ImmutableList.copyOf(previousMetadataLog), + ImmutableMap.of(TableMetadata.MAIN_BRANCH, SnapshotReference.builderForBranch(currentSnapshotId).build())); previousMetadataLog.add(latestPreviousMetadata); @@ -436,7 +445,8 @@ public void testAddPreviousMetadataRemoveMultiple() { TableMetadata.INITIAL_SORT_ORDER_ID, ImmutableList.of(SortOrder.unsorted()), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, - ImmutableList.copyOf(previousMetadataLog)); + ImmutableList.copyOf(previousMetadataLog), + ImmutableMap.of(TableMetadata.MAIN_BRANCH, SnapshotReference.builderForBranch(currentSnapshotId).build())); previousMetadataLog.add(latestPreviousMetadata); @@ -462,7 +472,7 @@ public void testV2UUIDValidation() { LAST_ASSIGNED_COLUMN_ID, 7, ImmutableList.of(TEST_SCHEMA), SPEC_5.specId(), ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, - ImmutableList.of(), ImmutableList.of(), ImmutableList.of()) + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), ImmutableMap.of()) ); } @@ -475,7 +485,7 @@ public void testVersionValidation() { System.currentTimeMillis(), LAST_ASSIGNED_COLUMN_ID, 7, ImmutableList.of(TEST_SCHEMA), SPEC_5.specId(), ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, - ImmutableList.of(), ImmutableList.of(), ImmutableList.of()) + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), ImmutableMap.of()) ); }