Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class BaseUpdatePartitionSpec implements UpdatePartitionSpec {
this.schema = spec.schema();
this.nameToField = indexSpecByName(spec);
this.transformToField = indexSpecByTransform(spec);
this.lastAssignedPartitionId = base.lastAssignedPartitionId();
this.lastAssignedPartitionId = base.lastPartitionId();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we need to rename the method, just the property in the metadata file.


spec.fields().stream()
.filter(field -> field.transform() instanceof UnknownTransform)
Expand Down
36 changes: 18 additions & 18 deletions core/src/main/java/org/apache/iceberg/TableMetadata.java
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ public String toString() {
private final Schema schema;
private final int defaultSpecId;
private final List<PartitionSpec> specs;
private final int lastAssignedPartitionId;
private final int lastPartitionId;
private final int defaultSortOrderId;
private final List<SortOrder> sortOrders;
private final Map<String, String> properties;
Expand All @@ -245,7 +245,7 @@ public String toString() {
Schema schema,
int defaultSpecId,
List<PartitionSpec> specs,
int lastAssignedPartitionId,
int lastPartitionId,
int defaultSortOrderId,
List<SortOrder> sortOrders,
Map<String, String> properties,
Expand Down Expand Up @@ -273,7 +273,7 @@ public String toString() {
this.schema = schema;
this.specs = specs;
this.defaultSpecId = defaultSpecId;
this.lastAssignedPartitionId = lastAssignedPartitionId;
this.lastPartitionId = lastPartitionId;
this.defaultSortOrderId = defaultSortOrderId;
this.sortOrders = sortOrders;
this.properties = properties;
Expand Down Expand Up @@ -378,8 +378,8 @@ public Map<Integer, PartitionSpec> specsById() {
return specsById;
}

int lastAssignedPartitionId() {
return lastAssignedPartitionId;
int lastPartitionId() {
return lastPartitionId;
}

public int defaultSpecId() {
Expand Down Expand Up @@ -451,7 +451,7 @@ public TableMetadata withUUID() {
return this;
} else {
return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location,
lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId,
lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, lastPartitionId,
defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog,
addPreviousFile(file, lastUpdatedMillis));
}
Expand All @@ -465,7 +465,7 @@ public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) {
List<SortOrder> updatedSortOrders = Lists.transform(sortOrders, order -> updateSortOrderSchema(newSchema, order));
return new TableMetadata(null, formatVersion, uuid, location,
lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, newSchema, defaultSpecId, updatedSpecs,
lastAssignedPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId,
lastPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId,
snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis));
}

Expand Down Expand Up @@ -500,7 +500,7 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) {

return new TableMetadata(null, formatVersion, uuid, location,
lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, newDefaultSpecId,
builder.build(), Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId()),
builder.build(), Math.max(lastPartitionId, newPartitionSpec.lastAssignedFieldId()),
defaultSortOrderId, sortOrders, properties,
currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis));
}
Expand Down Expand Up @@ -538,7 +538,7 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) {

return new TableMetadata(null, formatVersion, uuid, location,
lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs,
lastAssignedPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog,
lastPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog,
addPreviousFile(file, lastUpdatedMillis));
}

Expand All @@ -554,7 +554,7 @@ public TableMetadata addStagedSnapshot(Snapshot snapshot) {

return new TableMetadata(null, formatVersion, uuid, location,
snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs,
lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId,
lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId,
newSnapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis));
}

Expand All @@ -579,7 +579,7 @@ public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) {

return new TableMetadata(null, formatVersion, uuid, location,
snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs,
lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(),
lastPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(),
newSnapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis));
}

Expand Down Expand Up @@ -611,7 +611,7 @@ public TableMetadata removeSnapshotsIf(Predicate<Snapshot> removeIf) {

return new TableMetadata(null, formatVersion, uuid, location,
lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs,
lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered,
lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered,
ImmutableList.copyOf(newSnapshotLog), addPreviousFile(file, lastUpdatedMillis));
}

Expand All @@ -634,7 +634,7 @@ private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) {
.build();

return new TableMetadata(null, formatVersion, uuid, location,
lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId,
lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, lastPartitionId,
defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, newSnapshotLog,
addPreviousFile(file, lastUpdatedMillis));
}
Expand All @@ -643,7 +643,7 @@ public TableMetadata replaceProperties(Map<String, String> newProperties) {
ValidationException.check(newProperties != null, "Cannot set properties to null");
return new TableMetadata(null, formatVersion, uuid, location,
lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs,
lastAssignedPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots,
lastPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots,
snapshotLog, addPreviousFile(file, lastUpdatedMillis, newProperties));
}

Expand All @@ -662,7 +662,7 @@ public TableMetadata removeSnapshotLogEntries(Set<Long> snapshotIds) {

return new TableMetadata(null, formatVersion, uuid, location,
lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs,
lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId,
lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId,
snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis));
}

Expand Down Expand Up @@ -720,15 +720,15 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update

return new TableMetadata(null, formatVersion, uuid, newLocation,
lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchema,
specId, specListBuilder.build(), Math.max(lastAssignedPartitionId, freshSpec.lastAssignedFieldId()),
specId, specListBuilder.build(), Math.max(lastPartitionId, freshSpec.lastAssignedFieldId()),
orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties),
-1, snapshots, ImmutableList.of(), addPreviousFile(file, lastUpdatedMillis, newProperties));
}

public TableMetadata updateLocation(String newLocation) {
return new TableMetadata(null, formatVersion, uuid, newLocation,
lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs,
lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId,
lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId,
snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis));
}

Expand All @@ -745,7 +745,7 @@ public TableMetadata upgradeToFormatVersion(int newFormatVersion) {

return new TableMetadata(null, newFormatVersion, uuid, location,
lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs,
lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId,
lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId,
snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ private TableMetadataParser() {
static final String PARTITION_SPEC = "partition-spec";
static final String PARTITION_SPECS = "partition-specs";
static final String DEFAULT_SPEC_ID = "default-spec-id";
static final String LAST_ASSIGNED_PARTITION_ID = "last-assigned-partition-id";
static final String LAST_PARTITION_ID = "last-partition-id";
static final String DEFAULT_SORT_ORDER_ID = "default-sort-order-id";
static final String SORT_ORDERS = "sort-orders";
static final String PROPERTIES = "properties";
Expand Down Expand Up @@ -179,7 +179,7 @@ private static void toJson(TableMetadata metadata, JsonGenerator generator) thro
}
generator.writeEndArray();

generator.writeNumberField(LAST_ASSIGNED_PARTITION_ID, metadata.lastAssignedPartitionId());
generator.writeNumberField(LAST_PARTITION_ID, metadata.lastPartitionId());

generator.writeNumberField(DEFAULT_SORT_ORDER_ID, metadata.defaultSortOrderId());
generator.writeArrayFieldStart(SORT_ORDERS);
Expand Down Expand Up @@ -291,10 +291,10 @@ static TableMetadata fromJson(FileIO io, InputFile file, JsonNode node) {
schema, TableMetadata.INITIAL_SPEC_ID, node.get(PARTITION_SPEC)));
}

Integer lastAssignedPartitionId = JsonUtil.getIntOrNull(LAST_ASSIGNED_PARTITION_ID, node);
Integer lastAssignedPartitionId = JsonUtil.getIntOrNull(LAST_PARTITION_ID, node);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Won't this break backwards compatibility?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This property was just added, so the rename is safe.

if (lastAssignedPartitionId == null) {
Preconditions.checkArgument(formatVersion == 1,
"%s must exist in format v%s", LAST_ASSIGNED_PARTITION_ID, formatVersion);
"%s must exist in format v%s", LAST_PARTITION_ID, formatVersion);
lastAssignedPartitionId = specs.stream().mapToInt(PartitionSpec::lastAssignedFieldId).max().orElse(999);
}

Expand Down
10 changes: 5 additions & 5 deletions core/src/test/java/org/apache/iceberg/TestTableMetadata.java
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ public void testJsonConversion() throws Exception {
Assert.assertEquals("PartitionSpec map should match",
expected.specs(), metadata.specs());
Assert.assertEquals("lastAssignedFieldId across all PartitionSpecs should match",
expected.spec().lastAssignedFieldId(), metadata.lastAssignedPartitionId());
expected.spec().lastAssignedFieldId(), metadata.lastPartitionId());
Assert.assertEquals("Properties should match",
expected.properties(), metadata.properties());
Assert.assertEquals("Snapshot logs should match",
Expand Down Expand Up @@ -191,7 +191,7 @@ public void testBackwardCompat() throws Exception {
Assert.assertEquals("PartitionSpec should have ID TableMetadata.INITIAL_SPEC_ID",
TableMetadata.INITIAL_SPEC_ID, metadata.specs().get(0).specId());
Assert.assertEquals("lastAssignedFieldId across all PartitionSpecs should match",
expected.spec().lastAssignedFieldId(), metadata.lastAssignedPartitionId());
expected.spec().lastAssignedFieldId(), metadata.lastPartitionId());
Assert.assertEquals("Properties should match",
expected.properties(), metadata.properties());
Assert.assertEquals("Snapshot logs should match",
Expand Down Expand Up @@ -483,9 +483,9 @@ public void testParserV2PartitionSpecsValidation() throws Exception {

@Test
public void testParserV2LastAssignedFieldIdValidation() throws Exception {
String unsupportedVersion = readTableMetadataInputFile("TableMetadataV2MissingLastAssignedPartitionId.json");
AssertHelpers.assertThrows("Should reject v2 metadata without last assigned partition id",
IllegalArgumentException.class, "last-assigned-partition-id must exist in format v2",
String unsupportedVersion = readTableMetadataInputFile("TableMetadataV2MissingLastPartitionId.json");
AssertHelpers.assertThrows("Should reject v2 metadata without last assigned partition field id",
IllegalArgumentException.class, "last-partition-id must exist in format v2",
() -> TableMetadataParser.fromJson(
ops.io(), null, JsonUtil.mapper().readValue(unsupportedVersion, JsonNode.class))
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ public void testAddAfterLastFieldRemoved() {
.build(), table.spec());
V2Assert.assertEquals("Should match the last assigned field id",
999, table.spec().lastAssignedFieldId());
Assert.assertEquals(1000, table.ops().current().lastAssignedPartitionId());
Assert.assertEquals(1000, table.ops().current().lastPartitionId());

table.updateSpec()
.addField(bucket("id", 8))
Expand All @@ -230,6 +230,6 @@ public void testAddAfterLastFieldRemoved() {
.add(1, 1001, "id_bucket_8", "bucket[8]")
.build(), table.spec());
Assert.assertEquals(1001, table.spec().lastAssignedFieldId());
Assert.assertEquals(1001, table.ops().current().lastAssignedPartitionId());
Assert.assertEquals(1001, table.ops().current().lastPartitionId());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
]
}
],
"last-assigned-partition-id": 1000,
"last-partition-id": 1000,
"properties": {},
"current-snapshot-id": -1,
"snapshots": [],
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/resources/TableMetadataV2Valid.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
]
}
],
"last-assigned-partition-id": 1000,
"last-partition-id": 1000,
"default-sort-order-id": 3,
"sort-orders": [
{
Expand Down