diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index cb56389cbd09..ec8047e3353a 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -57,9 +57,14 @@ public class TableMetadata implements Serializable { static final int INITIAL_SPEC_ID = 0; static final int INITIAL_SORT_ORDER_ID = 1; static final int INITIAL_SCHEMA_ID = 0; + static final long INITIAL_SNAPSHOT_ID = -1; private static final long ONE_MINUTE = TimeUnit.MINUTES.toMillis(1); + public static TableMetadataUpdateBuilder builderFrom(TableMetadata base) { + return new TableMetadataUpdateBuilder(base); + } + public static TableMetadata newTableMetadata(Schema schema, PartitionSpec spec, SortOrder sortOrder, @@ -343,6 +348,10 @@ public String toString() { "Invalid table metadata: Cannot find current version"); } + InputFile file() { + return file; + } + public int formatVersion() { return formatVersion; } @@ -459,6 +468,10 @@ public Snapshot currentSnapshot() { return snapshotsById.get(currentSnapshotId); } + long currentSnapshotId() { + return currentSnapshotId; + } + public List snapshots() { return snapshots; } @@ -475,10 +488,7 @@ public TableMetadata withUUID() { if (uuid != null) { return this; } else { - return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, - lastSequenceNumber, lastUpdatedMillis, lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, - currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this).generateUUID().build(); } } @@ -500,11 +510,14 @@ public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) { builder.add(new Schema(newSchemaId, newSchema.columns(), newSchema.identifierFieldIds())); } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, - newSchemaId, builder.build(), defaultSpecId, updatedSpecs, lastAssignedPartitionId, - defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .refreshLastUpdateMillis() + .withLastColumnId(newLastColumnId) + .withSchemas(builder.build()) + .withCurrentSchemaId(newSchemaId) + .withSpecs(updatedSpecs) + .withSortOrders(updatedSortOrders) + .build(); } // The caller is responsible to pass a newPartitionSpec with correct partition field IDs @@ -538,11 +551,12 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { builder.add(freshSpec(newDefaultSpecId, schema, newPartitionSpec)); } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, newDefaultSpecId, - builder.build(), Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId()), - defaultSortOrderId, sortOrders, properties, - currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .refreshLastUpdateMillis() + .withDefaultSpecId(newDefaultSpecId) + .withSpecs(builder.build()) + .withLastAssignedPartitionId(Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId())) + .build(); } public TableMetadata replaceSortOrder(SortOrder newOrder) { @@ -577,10 +591,11 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { } } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .refreshLastUpdateMillis() + .withDefaultSortOrderId(newOrderId) + .withSortOrders(builder.build()) + .build(); } public TableMetadata addStagedSnapshot(Snapshot snapshot) { @@ -593,11 +608,11 @@ public TableMetadata addStagedSnapshot(Snapshot snapshot) { .add(snapshot) .build(); - return new TableMetadata(null, formatVersion, uuid, location, - snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, - currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, - defaultSortOrderId, sortOrders, properties, currentSnapshotId, newSnapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .withLastSequenceNumber(snapshot.sequenceNumber()) + .withLastUpdatedMillis(snapshot.timestampMillis()) + .withSnapshots(newSnapshots) + .build(); } public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { @@ -619,11 +634,13 @@ public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { .add(new SnapshotLogEntry(snapshot.timestampMillis(), snapshot.snapshotId())) .build(); - return new TableMetadata(null, formatVersion, uuid, location, - snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, - currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, - defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), newSnapshots, newSnapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .withLastSequenceNumber(snapshot.sequenceNumber()) + .withLastUpdatedMillis(snapshot.timestampMillis()) + .withCurrentSnapshotId(snapshot.snapshotId()) + .withSnapshots(newSnapshots) + .withSnapshotLog(newSnapshotLog) + .build(); } public TableMetadata removeSnapshotsIf(Predicate removeIf) { @@ -652,10 +669,11 @@ public TableMetadata removeSnapshotsIf(Predicate removeIf) { } } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, - ImmutableList.copyOf(newSnapshotLog), addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .refreshLastUpdateMillis() + .withSnapshots(filtered) + .withSnapshotLog(ImmutableList.copyOf(newSnapshotLog)) + .build(); } private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { @@ -676,26 +694,22 @@ private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { .add(new SnapshotLogEntry(nowMillis, snapshot.snapshotId())) .build(); - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, nowMillis, lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, - newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .refreshLastUpdateMillis() + .withCurrentSnapshotId(snapshot.snapshotId()) + .withSnapshotLog(newSnapshotLog) + .build(); } public TableMetadata replaceProperties(Map rawProperties) { ValidationException.check(rawProperties != null, "Cannot set properties to null"); Map newProperties = unreservedProperties(rawProperties); - TableMetadata metadata = new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, - snapshotLog, addPreviousFile(file, lastUpdatedMillis, newProperties)); - int newFormatVersion = PropertyUtil.propertyAsInt(rawProperties, TableProperties.FORMAT_VERSION, formatVersion); - if (formatVersion != newFormatVersion) { - metadata = metadata.upgradeToFormatVersion(newFormatVersion); - } - - return metadata; + return builderFrom(this) + .refreshLastUpdateMillis() + .withFormatVersion(newFormatVersion) + .withProperties(newProperties) + .build(); } public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { @@ -711,10 +725,10 @@ public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { Iterables.getLast(newSnapshotLog).snapshotId() == currentSnapshotId, "Cannot set invalid snapshot log: latest entry is not the current snapshot"); - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .refreshLastUpdateMillis() + .withSnapshotLog(newSnapshotLog) + .build(); } private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil.NextID nextID) { @@ -835,49 +849,41 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update schemasBuilder.add(new Schema(freshSchemaId, freshSchema.columns(), freshSchema.identifierFieldIds())); } - TableMetadata metadata = new TableMetadata(null, formatVersion, uuid, newLocation, - lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchemaId, schemasBuilder.build(), - specId, specListBuilder.build(), Math.max(lastAssignedPartitionId, newSpec.lastAssignedFieldId()), - orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties), - -1, snapshots, ImmutableList.of(), addPreviousFile(file, lastUpdatedMillis, newProperties)); - - if (formatVersion != newFormatVersion) { - metadata = metadata.upgradeToFormatVersion(newFormatVersion); - } - - return metadata; + return builderFrom(this) + .refreshLastUpdateMillis() + .withFormatVersion(newFormatVersion) + .withLocation(newLocation) + .withLastColumnId(newLastColumnId.get()) + .withCurrentSchemaId(freshSchemaId) + .withSchemas(schemasBuilder.build()) + .withDefaultSpecId(specId) + .withSpecs(specListBuilder.build()) + .withLastAssignedPartitionId(Math.max(lastAssignedPartitionId, newSpec.lastAssignedFieldId())) + .withDefaultSortOrderId(orderId) + .withSortOrders(sortOrdersBuilder.build()) + .withProperties(ImmutableMap.copyOf(newProperties)) + .withCurrentSnapshotId(INITIAL_SNAPSHOT_ID) + .withSnapshotLog(ImmutableList.of()) + .build(); } public TableMetadata updateLocation(String newLocation) { - return new TableMetadata(null, formatVersion, uuid, newLocation, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); + return builderFrom(this) + .refreshLastUpdateMillis() + .withLocation(newLocation) + .build(); } public TableMetadata upgradeToFormatVersion(int newFormatVersion) { - Preconditions.checkArgument(newFormatVersion <= SUPPORTED_TABLE_FORMAT_VERSION, - "Cannot upgrade table to unsupported format version: v%s (supported: v%s)", - newFormatVersion, SUPPORTED_TABLE_FORMAT_VERSION); - Preconditions.checkArgument(newFormatVersion >= formatVersion, - "Cannot downgrade v%s table to v%s", formatVersion, newFormatVersion); - - if (newFormatVersion == formatVersion) { - return this; - } - - return new TableMetadata(null, newFormatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); - } - - private List addPreviousFile(InputFile previousFile, long timestampMillis) { - return addPreviousFile(previousFile, timestampMillis, properties); + return builderFrom(this) + .refreshLastUpdateMillis() + .withFormatVersion(newFormatVersion) + .build(); } - private List addPreviousFile(InputFile previousFile, long timestampMillis, - Map updatedProperties) { + static List addPreviousFile(InputFile previousFile, long timestampMillis, + Map updatedProperties, + List previousFiles) { if (previousFile == null) { return previousFiles; } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataUpdateBuilder.java b/core/src/main/java/org/apache/iceberg/TableMetadataUpdateBuilder.java new file mode 100644 index 000000000000..7924801db1ba --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/TableMetadataUpdateBuilder.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +class TableMetadataUpdateBuilder { + + private InputFile file; + private Integer formatVersion; + private String uuid; + private String location; + private long lastSequenceNumber; + private long lastUpdatedMillis; + private int lastColumnId; + private int currentSchemaId; + private List schemas; + private int defaultSpecId; + private List specs; + private int lastAssignedPartitionId; + private int defaultSortOrderId; + private List sortOrders; + private Map properties; + private long currentSnapshotId; + private List snapshots; + private List snapshotLog; + private List previousFiles; + + private final int baseFormatVersion; + private final InputFile baseFile; + private final long baseLastUpdateMillis; + private final List basePreviousFiles; + + TableMetadataUpdateBuilder(TableMetadata base) { + this.formatVersion = base.formatVersion(); + this.uuid = base.uuid(); + this.location = base.location(); + this.lastSequenceNumber = base.lastSequenceNumber(); + this.lastUpdatedMillis = base.lastUpdatedMillis(); + this.lastColumnId = base.lastColumnId(); + this.currentSchemaId = base.currentSchemaId(); + this.schemas = base.schemas(); + this.specs = base.specs(); + this.defaultSpecId = base.defaultSpecId(); + this.lastAssignedPartitionId = base.lastAssignedPartitionId(); + this.defaultSortOrderId = base.defaultSortOrderId(); + this.sortOrders = base.sortOrders(); + this.properties = base.properties(); + this.currentSnapshotId = base.currentSnapshotId(); + this.snapshots = base.snapshots(); + this.snapshotLog = base.snapshotLog(); + + this.baseFormatVersion = base.formatVersion(); + this.baseFile = base.file(); + this.baseLastUpdateMillis = base.lastUpdatedMillis(); + this.basePreviousFiles = base.previousFiles(); + } + + public TableMetadataUpdateBuilder withFile(InputFile inputFile) { + this.file = inputFile; + return this; + } + + public TableMetadataUpdateBuilder withLocation(String inputLocation) { + this.location = inputLocation; + return this; + } + + public TableMetadataUpdateBuilder withFormatVersion(int inputFormatVersion) { + this.formatVersion = inputFormatVersion; + if (formatVersion != baseFormatVersion) { + Preconditions.checkArgument(formatVersion <= TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION, + "Cannot upgrade table to unsupported format version: v%s (supported: v%s)", + formatVersion, TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION); + Preconditions.checkArgument(formatVersion >= baseFormatVersion, + "Cannot downgrade v%s table to v%s", baseFormatVersion, formatVersion); + } + return this; + } + + public TableMetadataUpdateBuilder withUUID(String inputUuid) { + this.uuid = inputUuid; + return this; + } + + public TableMetadataUpdateBuilder generateUUID() { + return withUUID(UUID.randomUUID().toString()); + } + + public TableMetadataUpdateBuilder withLastSequenceNumber(long inputLastSequenceNumber) { + this.lastSequenceNumber = inputLastSequenceNumber; + return this; + } + + public TableMetadataUpdateBuilder refreshLastUpdateMillis() { + return withLastUpdatedMillis(System.currentTimeMillis()); + } + + public TableMetadataUpdateBuilder withLastUpdatedMillis(Long inputLastUpdatedMillis) { + this.lastUpdatedMillis = inputLastUpdatedMillis; + return this; + } + + public TableMetadataUpdateBuilder withLastColumnId(int inputLastColumnId) { + this.lastColumnId = inputLastColumnId; + return this; + } + + public TableMetadataUpdateBuilder withCurrentSchemaId(int inputCurrentSchemaId) { + this.currentSchemaId = inputCurrentSchemaId; + return this; + } + + public TableMetadataUpdateBuilder withSchemas(List inputSchemas) { + this.schemas = inputSchemas; + return this; + } + + public TableMetadataUpdateBuilder withDefaultSpecId(int inputDefaultSpecId) { + this.defaultSpecId = inputDefaultSpecId; + return this; + } + + public TableMetadataUpdateBuilder withSpecs(List inputSpecs) { + this.specs = inputSpecs; + return this; + } + + public TableMetadataUpdateBuilder withLastAssignedPartitionId(int inputLastAssignedPartitionId) { + this.lastAssignedPartitionId = inputLastAssignedPartitionId; + return this; + } + + public TableMetadataUpdateBuilder withDefaultSortOrderId(int inputDefaultSortOrderId) { + this.defaultSortOrderId = inputDefaultSortOrderId; + return this; + } + + public TableMetadataUpdateBuilder withSortOrders(List inputSortOrders) { + this.sortOrders = inputSortOrders; + return this; + } + + public TableMetadataUpdateBuilder withProperties(Map inputProperties) { + this.properties = inputProperties; + return this; + } + + public TableMetadataUpdateBuilder withCurrentSnapshotId(long inputCurrentSnapshotId) { + this.currentSnapshotId = inputCurrentSnapshotId; + return this; + } + + public TableMetadataUpdateBuilder withSnapshots(List inputSnapshots) { + this.snapshots = inputSnapshots; + return this; + } + + public TableMetadataUpdateBuilder withSnapshotLog(List inputSnapshotLog) { + this.snapshotLog = inputSnapshotLog; + return this; + } + + public TableMetadataUpdateBuilder withPreviousFiles(List inputPreviousFiles) { + this.previousFiles = inputPreviousFiles; + return this; + } + + public TableMetadata build() { + if (previousFiles == null) { + previousFiles = TableMetadata.addPreviousFile(baseFile, baseLastUpdateMillis, properties, basePreviousFiles); + } + + return new TableMetadata(file, formatVersion, uuid, location, lastSequenceNumber, lastUpdatedMillis, + lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, + defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, previousFiles); + } +}