From 5b42e92823a4413a7fa8b4b5544e7f487c1f95fc Mon Sep 17 00:00:00 2001 From: Jun He Date: Sun, 31 Jan 2021 11:27:41 -0800 Subject: [PATCH 1/2] rename last-assigned-partition-id to last-partition-id in table metadata --- .../iceberg/BaseUpdatePartitionSpec.java | 2 +- .../org/apache/iceberg/TableMetadata.java | 36 +++++++++---------- .../apache/iceberg/TableMetadataParser.java | 8 ++--- .../org/apache/iceberg/TestTableMetadata.java | 10 +++--- .../iceberg/TestTableUpdatePartitionSpec.java | 4 +-- ...ableMetadataV2MissingLastPartitionId.json} | 0 .../TableMetadataV2MissingSortOrder.json | 2 +- .../test/resources/TableMetadataV2Valid.json | 2 +- 8 files changed, 32 insertions(+), 32 deletions(-) rename core/src/test/resources/{TableMetadataV2MissingLastAssignedPartitionId.json => TableMetadataV2MissingLastPartitionId.json} (100%) diff --git a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java index e1fe84be1dc3..009894a069b7 100644 --- a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java +++ b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java @@ -69,7 +69,7 @@ class BaseUpdatePartitionSpec implements UpdatePartitionSpec { this.schema = spec.schema(); this.nameToField = indexSpecByName(spec); this.transformToField = indexSpecByTransform(spec); - this.lastAssignedPartitionId = base.lastAssignedPartitionId(); + this.lastAssignedPartitionId = base.lastPartitionId(); spec.fields().stream() .filter(field -> field.transform() instanceof UnknownTransform) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index cf8e64b0203d..713ac68dd89c 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -222,7 +222,7 @@ public String toString() { private final Schema schema; private final int defaultSpecId; private final List specs; - private final int lastAssignedPartitionId; + private final int lastPartitionId; private final int defaultSortOrderId; private final List sortOrders; private final Map properties; @@ -245,7 +245,7 @@ public String toString() { Schema schema, int defaultSpecId, List specs, - int lastAssignedPartitionId, + int lastPartitionId, int defaultSortOrderId, List sortOrders, Map properties, @@ -273,7 +273,7 @@ public String toString() { this.schema = schema; this.specs = specs; this.defaultSpecId = defaultSpecId; - this.lastAssignedPartitionId = lastAssignedPartitionId; + this.lastPartitionId = lastPartitionId; this.defaultSortOrderId = defaultSortOrderId; this.sortOrders = sortOrders; this.properties = properties; @@ -378,8 +378,8 @@ public Map specsById() { return specsById; } - int lastAssignedPartitionId() { - return lastAssignedPartitionId; + int lastPartitionId() { + return lastPartitionId; } public int defaultSpecId() { @@ -451,7 +451,7 @@ public TableMetadata withUUID() { return this; } else { return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, - lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId, + lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -465,7 +465,7 @@ public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) { List updatedSortOrders = Lists.transform(sortOrders, order -> updateSortOrderSchema(newSchema, order)); return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, newSchema, defaultSpecId, updatedSpecs, - lastAssignedPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, + lastPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -500,7 +500,7 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, newDefaultSpecId, - builder.build(), Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId()), + builder.build(), Math.max(lastPartitionId, newPartitionSpec.lastAssignedFieldId()), defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -538,7 +538,7 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, + lastPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -554,7 +554,7 @@ public TableMetadata addStagedSnapshot(Snapshot snapshot) { return new TableMetadata(null, formatVersion, uuid, location, snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, newSnapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -579,7 +579,7 @@ public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { return new TableMetadata(null, formatVersion, uuid, location, snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), + lastPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), newSnapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -611,7 +611,7 @@ public TableMetadata removeSnapshotsIf(Predicate removeIf) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, + lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, ImmutableList.copyOf(newSnapshotLog), addPreviousFile(file, lastUpdatedMillis)); } @@ -634,7 +634,7 @@ private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { .build(); return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId, + lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, lastPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -643,7 +643,7 @@ public TableMetadata replaceProperties(Map newProperties) { ValidationException.check(newProperties != null, "Cannot set properties to null"); return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, + lastPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis, newProperties)); } @@ -662,7 +662,7 @@ public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -720,7 +720,7 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update return new TableMetadata(null, formatVersion, uuid, newLocation, lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchema, - specId, specListBuilder.build(), Math.max(lastAssignedPartitionId, freshSpec.lastAssignedFieldId()), + specId, specListBuilder.build(), Math.max(lastPartitionId, freshSpec.lastAssignedFieldId()), orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties), -1, snapshots, ImmutableList.of(), addPreviousFile(file, lastUpdatedMillis, newProperties)); } @@ -728,7 +728,7 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update public TableMetadata updateLocation(String newLocation) { return new TableMetadata(null, formatVersion, uuid, newLocation, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -745,7 +745,7 @@ public TableMetadata upgradeToFormatVersion(int newFormatVersion) { return new TableMetadata(null, newFormatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index b27440e26649..59404b3908a4 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -91,7 +91,7 @@ private TableMetadataParser() { static final String PARTITION_SPEC = "partition-spec"; static final String PARTITION_SPECS = "partition-specs"; static final String DEFAULT_SPEC_ID = "default-spec-id"; - static final String LAST_ASSIGNED_PARTITION_ID = "last-assigned-partition-id"; + static final String LAST_PARTITION_ID = "last-partition-id"; static final String DEFAULT_SORT_ORDER_ID = "default-sort-order-id"; static final String SORT_ORDERS = "sort-orders"; static final String PROPERTIES = "properties"; @@ -179,7 +179,7 @@ private static void toJson(TableMetadata metadata, JsonGenerator generator) thro } generator.writeEndArray(); - generator.writeNumberField(LAST_ASSIGNED_PARTITION_ID, metadata.lastAssignedPartitionId()); + generator.writeNumberField(LAST_PARTITION_ID, metadata.lastPartitionId()); generator.writeNumberField(DEFAULT_SORT_ORDER_ID, metadata.defaultSortOrderId()); generator.writeArrayFieldStart(SORT_ORDERS); @@ -291,10 +291,10 @@ static TableMetadata fromJson(FileIO io, InputFile file, JsonNode node) { schema, TableMetadata.INITIAL_SPEC_ID, node.get(PARTITION_SPEC))); } - Integer lastAssignedPartitionId = JsonUtil.getIntOrNull(LAST_ASSIGNED_PARTITION_ID, node); + Integer lastAssignedPartitionId = JsonUtil.getIntOrNull(LAST_PARTITION_ID, node); if (lastAssignedPartitionId == null) { Preconditions.checkArgument(formatVersion == 1, - "%s must exist in format v%s", LAST_ASSIGNED_PARTITION_ID, formatVersion); + "%s must exist in format v%s", LAST_PARTITION_ID, formatVersion); lastAssignedPartitionId = specs.stream().mapToInt(PartitionSpec::lastAssignedFieldId).max().orElse(999); } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 1c4040f0ecaf..6bc82bb2920d 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -128,7 +128,7 @@ public void testJsonConversion() throws Exception { Assert.assertEquals("PartitionSpec map should match", expected.specs(), metadata.specs()); Assert.assertEquals("lastAssignedFieldId across all PartitionSpecs should match", - expected.spec().lastAssignedFieldId(), metadata.lastAssignedPartitionId()); + expected.spec().lastAssignedFieldId(), metadata.lastPartitionId()); Assert.assertEquals("Properties should match", expected.properties(), metadata.properties()); Assert.assertEquals("Snapshot logs should match", @@ -191,7 +191,7 @@ public void testBackwardCompat() throws Exception { Assert.assertEquals("PartitionSpec should have ID TableMetadata.INITIAL_SPEC_ID", TableMetadata.INITIAL_SPEC_ID, metadata.specs().get(0).specId()); Assert.assertEquals("lastAssignedFieldId across all PartitionSpecs should match", - expected.spec().lastAssignedFieldId(), metadata.lastAssignedPartitionId()); + expected.spec().lastAssignedFieldId(), metadata.lastPartitionId()); Assert.assertEquals("Properties should match", expected.properties(), metadata.properties()); Assert.assertEquals("Snapshot logs should match", @@ -483,9 +483,9 @@ public void testParserV2PartitionSpecsValidation() throws Exception { @Test public void testParserV2LastAssignedFieldIdValidation() throws Exception { - String unsupportedVersion = readTableMetadataInputFile("TableMetadataV2MissingLastAssignedPartitionId.json"); - AssertHelpers.assertThrows("Should reject v2 metadata without last assigned partition id", - IllegalArgumentException.class, "last-assigned-partition-id must exist in format v2", + String unsupportedVersion = readTableMetadataInputFile("TableMetadataV2MissingLastPartitionId.json"); + AssertHelpers.assertThrows("Should reject v2 metadata without last assigned partition field id", + IllegalArgumentException.class, "last-partition-id must exist in format v2", () -> TableMetadataParser.fromJson( ops.io(), null, JsonUtil.mapper().readValue(unsupportedVersion, JsonNode.class)) ); diff --git a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java index 1e5bd41af0a0..e08121332a20 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java @@ -214,7 +214,7 @@ public void testAddAfterLastFieldRemoved() { .build(), table.spec()); V2Assert.assertEquals("Should match the last assigned field id", 999, table.spec().lastAssignedFieldId()); - Assert.assertEquals(1000, table.ops().current().lastAssignedPartitionId()); + Assert.assertEquals(1000, table.ops().current().lastPartitionId()); table.updateSpec() .addField(bucket("id", 8)) @@ -230,6 +230,6 @@ public void testAddAfterLastFieldRemoved() { .add(1, 1001, "id_bucket_8", "bucket[8]") .build(), table.spec()); Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); - Assert.assertEquals(1001, table.ops().current().lastAssignedPartitionId()); + Assert.assertEquals(1001, table.ops().current().lastPartitionId()); } } diff --git a/core/src/test/resources/TableMetadataV2MissingLastAssignedPartitionId.json b/core/src/test/resources/TableMetadataV2MissingLastPartitionId.json similarity index 100% rename from core/src/test/resources/TableMetadataV2MissingLastAssignedPartitionId.json rename to core/src/test/resources/TableMetadataV2MissingLastPartitionId.json diff --git a/core/src/test/resources/TableMetadataV2MissingSortOrder.json b/core/src/test/resources/TableMetadataV2MissingSortOrder.json index f656e08ff5ad..93e6f879d4f0 100644 --- a/core/src/test/resources/TableMetadataV2MissingSortOrder.json +++ b/core/src/test/resources/TableMetadataV2MissingSortOrder.json @@ -43,7 +43,7 @@ ] } ], - "last-assigned-partition-id": 1000, + "last-partition-id": 1000, "properties": {}, "current-snapshot-id": -1, "snapshots": [], diff --git a/core/src/test/resources/TableMetadataV2Valid.json b/core/src/test/resources/TableMetadataV2Valid.json index aee3de6c360f..fabd9726561c 100644 --- a/core/src/test/resources/TableMetadataV2Valid.json +++ b/core/src/test/resources/TableMetadataV2Valid.json @@ -43,7 +43,7 @@ ] } ], - "last-assigned-partition-id": 1000, + "last-partition-id": 1000, "default-sort-order-id": 3, "sort-orders": [ { From 4ee3638cf077244c91a64c14e838ac8d2686d7eb Mon Sep 17 00:00:00 2001 From: Jun He Date: Mon, 1 Feb 2021 11:14:29 -0800 Subject: [PATCH 2/2] roll back the field and and method name changes in other classes --- .../iceberg/BaseUpdatePartitionSpec.java | 2 +- .../org/apache/iceberg/TableMetadata.java | 36 +++++++++---------- .../apache/iceberg/TableMetadataParser.java | 2 +- .../org/apache/iceberg/TestTableMetadata.java | 4 +-- .../iceberg/TestTableUpdatePartitionSpec.java | 4 +-- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java index 009894a069b7..e1fe84be1dc3 100644 --- a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java +++ b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java @@ -69,7 +69,7 @@ class BaseUpdatePartitionSpec implements UpdatePartitionSpec { this.schema = spec.schema(); this.nameToField = indexSpecByName(spec); this.transformToField = indexSpecByTransform(spec); - this.lastAssignedPartitionId = base.lastPartitionId(); + this.lastAssignedPartitionId = base.lastAssignedPartitionId(); spec.fields().stream() .filter(field -> field.transform() instanceof UnknownTransform) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 713ac68dd89c..cf8e64b0203d 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -222,7 +222,7 @@ public String toString() { private final Schema schema; private final int defaultSpecId; private final List specs; - private final int lastPartitionId; + private final int lastAssignedPartitionId; private final int defaultSortOrderId; private final List sortOrders; private final Map properties; @@ -245,7 +245,7 @@ public String toString() { Schema schema, int defaultSpecId, List specs, - int lastPartitionId, + int lastAssignedPartitionId, int defaultSortOrderId, List sortOrders, Map properties, @@ -273,7 +273,7 @@ public String toString() { this.schema = schema; this.specs = specs; this.defaultSpecId = defaultSpecId; - this.lastPartitionId = lastPartitionId; + this.lastAssignedPartitionId = lastAssignedPartitionId; this.defaultSortOrderId = defaultSortOrderId; this.sortOrders = sortOrders; this.properties = properties; @@ -378,8 +378,8 @@ public Map specsById() { return specsById; } - int lastPartitionId() { - return lastPartitionId; + int lastAssignedPartitionId() { + return lastAssignedPartitionId; } public int defaultSpecId() { @@ -451,7 +451,7 @@ public TableMetadata withUUID() { return this; } else { return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, - lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, lastPartitionId, + lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -465,7 +465,7 @@ public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) { List updatedSortOrders = Lists.transform(sortOrders, order -> updateSortOrderSchema(newSchema, order)); return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, newSchema, defaultSpecId, updatedSpecs, - lastPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, + lastAssignedPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -500,7 +500,7 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, newDefaultSpecId, - builder.build(), Math.max(lastPartitionId, newPartitionSpec.lastAssignedFieldId()), + builder.build(), Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId()), defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -538,7 +538,7 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, + lastAssignedPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -554,7 +554,7 @@ public TableMetadata addStagedSnapshot(Snapshot snapshot) { return new TableMetadata(null, formatVersion, uuid, location, snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, - lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, newSnapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -579,7 +579,7 @@ public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { return new TableMetadata(null, formatVersion, uuid, location, snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, - lastPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), newSnapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -611,7 +611,7 @@ public TableMetadata removeSnapshotsIf(Predicate removeIf) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, ImmutableList.copyOf(newSnapshotLog), addPreviousFile(file, lastUpdatedMillis)); } @@ -634,7 +634,7 @@ private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { .build(); return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, lastPartitionId, + lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -643,7 +643,7 @@ public TableMetadata replaceProperties(Map newProperties) { ValidationException.check(newProperties != null, "Cannot set properties to null"); return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis, newProperties)); } @@ -662,7 +662,7 @@ public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -720,7 +720,7 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update return new TableMetadata(null, formatVersion, uuid, newLocation, lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchema, - specId, specListBuilder.build(), Math.max(lastPartitionId, freshSpec.lastAssignedFieldId()), + specId, specListBuilder.build(), Math.max(lastAssignedPartitionId, freshSpec.lastAssignedFieldId()), orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties), -1, snapshots, ImmutableList.of(), addPreviousFile(file, lastUpdatedMillis, newProperties)); } @@ -728,7 +728,7 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update public TableMetadata updateLocation(String newLocation) { return new TableMetadata(null, formatVersion, uuid, newLocation, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -745,7 +745,7 @@ public TableMetadata upgradeToFormatVersion(int newFormatVersion) { return new TableMetadata(null, newFormatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - lastPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index 59404b3908a4..1cb997554827 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -179,7 +179,7 @@ private static void toJson(TableMetadata metadata, JsonGenerator generator) thro } generator.writeEndArray(); - generator.writeNumberField(LAST_PARTITION_ID, metadata.lastPartitionId()); + generator.writeNumberField(LAST_PARTITION_ID, metadata.lastAssignedPartitionId()); generator.writeNumberField(DEFAULT_SORT_ORDER_ID, metadata.defaultSortOrderId()); generator.writeArrayFieldStart(SORT_ORDERS); diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 6bc82bb2920d..62b35f84e038 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -128,7 +128,7 @@ public void testJsonConversion() throws Exception { Assert.assertEquals("PartitionSpec map should match", expected.specs(), metadata.specs()); Assert.assertEquals("lastAssignedFieldId across all PartitionSpecs should match", - expected.spec().lastAssignedFieldId(), metadata.lastPartitionId()); + expected.spec().lastAssignedFieldId(), metadata.lastAssignedPartitionId()); Assert.assertEquals("Properties should match", expected.properties(), metadata.properties()); Assert.assertEquals("Snapshot logs should match", @@ -191,7 +191,7 @@ public void testBackwardCompat() throws Exception { Assert.assertEquals("PartitionSpec should have ID TableMetadata.INITIAL_SPEC_ID", TableMetadata.INITIAL_SPEC_ID, metadata.specs().get(0).specId()); Assert.assertEquals("lastAssignedFieldId across all PartitionSpecs should match", - expected.spec().lastAssignedFieldId(), metadata.lastPartitionId()); + expected.spec().lastAssignedFieldId(), metadata.lastAssignedPartitionId()); Assert.assertEquals("Properties should match", expected.properties(), metadata.properties()); Assert.assertEquals("Snapshot logs should match", diff --git a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java index e08121332a20..1e5bd41af0a0 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java @@ -214,7 +214,7 @@ public void testAddAfterLastFieldRemoved() { .build(), table.spec()); V2Assert.assertEquals("Should match the last assigned field id", 999, table.spec().lastAssignedFieldId()); - Assert.assertEquals(1000, table.ops().current().lastPartitionId()); + Assert.assertEquals(1000, table.ops().current().lastAssignedPartitionId()); table.updateSpec() .addField(bucket("id", 8)) @@ -230,6 +230,6 @@ public void testAddAfterLastFieldRemoved() { .add(1, 1001, "id_bucket_8", "bucket[8]") .build(), table.spec()); Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); - Assert.assertEquals(1001, table.ops().current().lastPartitionId()); + Assert.assertEquals(1001, table.ops().current().lastAssignedPartitionId()); } }