diff --git a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java index 697ecc43bbef..e1fe84be1dc3 100644 --- a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java +++ b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java @@ -69,8 +69,7 @@ class BaseUpdatePartitionSpec implements UpdatePartitionSpec { this.schema = spec.schema(); this.nameToField = indexSpecByName(spec); this.transformToField = indexSpecByTransform(spec); - this.lastAssignedPartitionId = - base.specs().stream().mapToInt(PartitionSpec::lastAssignedFieldId).max().orElse(999); + this.lastAssignedPartitionId = base.lastAssignedPartitionId(); spec.fields().stream() .filter(field -> field.transform() instanceof UnknownTransform) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 7aa23c87b48a..f6f17ffe5da1 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -113,7 +113,7 @@ static TableMetadata newTableMetadata(Schema schema, return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, INITIAL_SEQUENCE_NUMBER, System.currentTimeMillis(), lastColumnId.get(), freshSchema, INITIAL_SPEC_ID, ImmutableList.of(freshSpec), - freshSortOrderId, ImmutableList.of(freshSortOrder), + freshSpec.lastAssignedFieldId(), freshSortOrderId, ImmutableList.of(freshSortOrder), ImmutableMap.copyOf(properties), -1, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()); } @@ -218,6 +218,7 @@ public String toString() { private final Schema schema; private final int defaultSpecId; private final List specs; + private final int lastAssignedPartitionId; private final int defaultSortOrderId; private final List sortOrders; private final Map properties; @@ -240,6 +241,7 @@ public String toString() { Schema schema, int defaultSpecId, List specs, + int lastAssignedPartitionId, int defaultSortOrderId, List sortOrders, Map properties, @@ -267,6 +269,7 @@ public String toString() { this.schema = schema; this.specs = specs; this.defaultSpecId = defaultSpecId; + this.lastAssignedPartitionId = lastAssignedPartitionId; this.defaultSortOrderId = defaultSortOrderId; this.sortOrders = sortOrders; this.properties = properties; @@ -371,6 +374,10 @@ public Map specsById() { return specsById; } + int lastAssignedPartitionId() { + return lastAssignedPartitionId; + } + public int defaultSpecId() { return defaultSpecId; } @@ -440,7 +447,7 @@ public TableMetadata withUUID() { return this; } else { return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, - lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, + lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -454,8 +461,8 @@ public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) { List updatedSortOrders = Lists.transform(sortOrders, order -> updateSortOrderSchema(newSchema, order)); return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, newSchema, defaultSpecId, updatedSpecs, - defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + lastAssignedPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, + snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } // The caller is responsible to pass a newPartitionSpec with correct partition field IDs @@ -489,7 +496,8 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, newDefaultSpecId, - builder.build(), defaultSortOrderId, sortOrders, properties, + builder.build(), Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId()), + defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -526,7 +534,7 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, + lastAssignedPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -542,8 +550,8 @@ public TableMetadata addStagedSnapshot(Snapshot snapshot) { return new TableMetadata(null, formatVersion, uuid, location, snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, - defaultSortOrderId, sortOrders, properties, currentSnapshotId, newSnapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + newSnapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { @@ -567,8 +575,8 @@ public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { return new TableMetadata(null, formatVersion, uuid, location, snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, - defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), newSnapshots, newSnapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), + newSnapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } public TableMetadata removeSnapshotsIf(Predicate removeIf) { @@ -599,7 +607,7 @@ public TableMetadata removeSnapshotsIf(Predicate removeIf) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, ImmutableList.copyOf(newSnapshotLog), addPreviousFile(file, lastUpdatedMillis)); } @@ -622,8 +630,8 @@ private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { .build(); return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, defaultSortOrderId, - sortOrders, properties, snapshot.snapshotId(), snapshots, newSnapshotLog, + lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId, + defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -631,8 +639,8 @@ public TableMetadata replaceProperties(Map newProperties) { ValidationException.check(newProperties != null, "Cannot set properties to null"); return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis, newProperties)); + lastAssignedPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, + snapshotLog, addPreviousFile(file, lastUpdatedMillis, newProperties)); } public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { @@ -650,8 +658,8 @@ public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { return new TableMetadata(null, formatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, newSnapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } // The caller is responsible to pass a updatedPartitionSpec with correct partition field IDs @@ -708,15 +716,16 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update return new TableMetadata(null, formatVersion, uuid, newLocation, lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchema, - specId, specListBuilder.build(), orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties), + specId, specListBuilder.build(), Math.max(lastAssignedPartitionId, freshSpec.lastAssignedFieldId()), + orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties), -1, snapshots, ImmutableList.of(), addPreviousFile(file, lastUpdatedMillis, newProperties)); } public TableMetadata updateLocation(String newLocation) { return new TableMetadata(null, formatVersion, uuid, newLocation, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } public TableMetadata upgradeToFormatVersion(int newFormatVersion) { @@ -732,8 +741,8 @@ public TableMetadata upgradeToFormatVersion(int newFormatVersion) { return new TableMetadata(null, newFormatVersion, uuid, location, lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, - defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } private List addPreviousFile(InputFile previousFile, long timestampMillis) { diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index 58805a320415..b27440e26649 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -91,6 +91,7 @@ private TableMetadataParser() { static final String PARTITION_SPEC = "partition-spec"; static final String PARTITION_SPECS = "partition-specs"; static final String DEFAULT_SPEC_ID = "default-spec-id"; + static final String LAST_ASSIGNED_PARTITION_ID = "last-assigned-partition-id"; static final String DEFAULT_SORT_ORDER_ID = "default-sort-order-id"; static final String SORT_ORDERS = "sort-orders"; static final String PROPERTIES = "properties"; @@ -178,6 +179,8 @@ private static void toJson(TableMetadata metadata, JsonGenerator generator) thro } generator.writeEndArray(); + generator.writeNumberField(LAST_ASSIGNED_PARTITION_ID, metadata.lastAssignedPartitionId()); + generator.writeNumberField(DEFAULT_SORT_ORDER_ID, metadata.defaultSortOrderId()); generator.writeArrayFieldStart(SORT_ORDERS); for (SortOrder sortOrder : metadata.sortOrders()) { @@ -288,6 +291,13 @@ static TableMetadata fromJson(FileIO io, InputFile file, JsonNode node) { schema, TableMetadata.INITIAL_SPEC_ID, node.get(PARTITION_SPEC))); } + Integer lastAssignedPartitionId = JsonUtil.getIntOrNull(LAST_ASSIGNED_PARTITION_ID, node); + if (lastAssignedPartitionId == null) { + Preconditions.checkArgument(formatVersion == 1, + "%s must exist in format v%s", LAST_ASSIGNED_PARTITION_ID, formatVersion); + lastAssignedPartitionId = specs.stream().mapToInt(PartitionSpec::lastAssignedFieldId).max().orElse(999); + } + JsonNode sortOrderArray = node.get(SORT_ORDERS); List sortOrders; int defaultSortOrderId; @@ -342,7 +352,7 @@ static TableMetadata fromJson(FileIO io, InputFile file, JsonNode node) { return new TableMetadata(file, formatVersion, uuid, location, lastSequenceNumber, lastUpdatedMillis, lastAssignedColumnId, schema, defaultSpecId, specs, - defaultSortOrderId, sortOrders, properties, currentVersionId, snapshots, entries.build(), - metadataEntries.build()); + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentVersionId, + snapshots, entries.build(), metadataEntries.build()); } } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 7509ac4d667a..1c4040f0ecaf 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -101,7 +101,7 @@ public void testJsonConversion() throws Exception { .build(); TableMetadata expected = new TableMetadata(null, 2, UUID.randomUUID().toString(), TEST_LOCATION, - SEQ_NO, System.currentTimeMillis(), 3, TEST_SCHEMA, 5, ImmutableList.of(SPEC_5), + SEQ_NO, System.currentTimeMillis(), 3, TEST_SCHEMA, 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), snapshotLog, ImmutableList.of()); @@ -127,6 +127,8 @@ public void testJsonConversion() throws Exception { expected.defaultSpecId(), metadata.defaultSpecId()); Assert.assertEquals("PartitionSpec map should match", expected.specs(), metadata.specs()); + Assert.assertEquals("lastAssignedFieldId across all PartitionSpecs should match", + expected.spec().lastAssignedFieldId(), metadata.lastAssignedPartitionId()); Assert.assertEquals("Properties should match", expected.properties(), metadata.properties()); Assert.assertEquals("Snapshot logs should match", @@ -159,7 +161,7 @@ public void testBackwardCompat() throws Exception { new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), spec.specId()))); TableMetadata expected = new TableMetadata(null, 1, null, TEST_LOCATION, - 0, System.currentTimeMillis(), 3, TEST_SCHEMA, 6, ImmutableList.of(spec), + 0, System.currentTimeMillis(), 3, TEST_SCHEMA, 6, ImmutableList.of(spec), spec.lastAssignedFieldId(), TableMetadata.INITIAL_SORT_ORDER_ID, ImmutableList.of(sortOrder), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), ImmutableList.of(), ImmutableList.of()); @@ -188,6 +190,8 @@ public void testBackwardCompat() throws Exception { metadata.specs().get(0).compatibleWith(spec)); Assert.assertEquals("PartitionSpec should have ID TableMetadata.INITIAL_SPEC_ID", TableMetadata.INITIAL_SPEC_ID, metadata.specs().get(0).specId()); + Assert.assertEquals("lastAssignedFieldId across all PartitionSpecs should match", + expected.spec().lastAssignedFieldId(), metadata.lastAssignedPartitionId()); Assert.assertEquals("Properties should match", expected.properties(), metadata.properties()); Assert.assertEquals("Snapshot logs should match", @@ -269,7 +273,7 @@ public void testJsonWithPreviousMetadataLog() throws Exception { "/tmp/000001-" + UUID.randomUUID().toString() + ".metadata.json")); TableMetadata base = new TableMetadata(null, 1, UUID.randomUUID().toString(), TEST_LOCATION, - 0, System.currentTimeMillis(), 3, TEST_SCHEMA, 5, ImmutableList.of(SPEC_5), + 0, System.currentTimeMillis(), 3, TEST_SCHEMA, 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, ImmutableList.copyOf(previousMetadataLog)); @@ -305,6 +309,7 @@ public void testAddPreviousMetadataRemoveNone() { TableMetadata base = new TableMetadata(localInput(latestPreviousMetadata.file()), 1, UUID.randomUUID().toString(), TEST_LOCATION, 0, currentTimestamp - 80, 3, TEST_SCHEMA, 5, ImmutableList.of(SPEC_5), + SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, ImmutableList.copyOf(previousMetadataLog)); @@ -350,7 +355,7 @@ public void testAddPreviousMetadataRemoveOne() { TableMetadata base = new TableMetadata(localInput(latestPreviousMetadata.file()), 1, UUID.randomUUID().toString(), TEST_LOCATION, 0, currentTimestamp - 50, 3, TEST_SCHEMA, 5, - ImmutableList.of(SPEC_5), 3, ImmutableList.of(SORT_ORDER_3), + ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, ImmutableList.copyOf(previousMetadataLog)); @@ -401,8 +406,8 @@ public void testAddPreviousMetadataRemoveMultiple() { TableMetadata base = new TableMetadata(localInput(latestPreviousMetadata.file()), 1, UUID.randomUUID().toString(), TEST_LOCATION, 0, currentTimestamp - 50, 3, TEST_SCHEMA, 2, - ImmutableList.of(SPEC_5), TableMetadata.INITIAL_SORT_ORDER_ID, ImmutableList.of(SortOrder.unsorted()), - ImmutableMap.of("property", "value"), currentSnapshotId, + ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), TableMetadata.INITIAL_SORT_ORDER_ID, + ImmutableList.of(SortOrder.unsorted()), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, ImmutableList.copyOf(previousMetadataLog)); @@ -428,7 +433,7 @@ public void testV2UUIDValidation() { IllegalArgumentException.class, "UUID is required in format v2", () -> new TableMetadata(null, 2, null, TEST_LOCATION, SEQ_NO, System.currentTimeMillis(), LAST_ASSIGNED_COLUMN_ID, TEST_SCHEMA, SPEC_5.specId(), ImmutableList.of(SPEC_5), - 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, + SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()) ); } @@ -440,7 +445,7 @@ public void testVersionValidation() { IllegalArgumentException.class, "Unsupported format version: v" + unsupportedVersion, () -> new TableMetadata(null, unsupportedVersion, null, TEST_LOCATION, SEQ_NO, System.currentTimeMillis(), LAST_ASSIGNED_COLUMN_ID, TEST_SCHEMA, SPEC_5.specId(), ImmutableList.of(SPEC_5), - 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, + SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()) ); } @@ -476,6 +481,16 @@ public void testParserV2PartitionSpecsValidation() throws Exception { ); } + @Test + public void testParserV2LastAssignedFieldIdValidation() throws Exception { + String unsupportedVersion = readTableMetadataInputFile("TableMetadataV2MissingLastAssignedPartitionId.json"); + AssertHelpers.assertThrows("Should reject v2 metadata without last assigned partition id", + IllegalArgumentException.class, "last-assigned-partition-id must exist in format v2", + () -> TableMetadataParser.fromJson( + ops.io(), null, JsonUtil.mapper().readValue(unsupportedVersion, JsonNode.class)) + ); + } + @Test public void testParserV2SortOrderValidation() throws Exception { String unsupportedVersion = readTableMetadataInputFile("TableMetadataV2MissingSortOrder.json"); diff --git a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java index 76f9fa0b28fc..1e5bd41af0a0 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java @@ -203,6 +203,19 @@ public void testAddAfterLastFieldRemoved() { .removeField("data_bucket") .commit(); + V1Assert.assertEquals("Should add a new id bucket", PartitionSpec.builderFor(table.schema()) + .withSpecId(1) + .alwaysNull("data", "data_bucket") + .build(), table.spec()); + V1Assert.assertEquals("Should match the last assigned field id", + 1000, table.spec().lastAssignedFieldId()); + V2Assert.assertEquals("Should add a new id bucket", PartitionSpec.builderFor(table.schema()) + .withSpecId(1) + .build(), table.spec()); + V2Assert.assertEquals("Should match the last assigned field id", + 999, table.spec().lastAssignedFieldId()); + Assert.assertEquals(1000, table.ops().current().lastAssignedPartitionId()); + table.updateSpec() .addField(bucket("id", 8)) .commit(); @@ -217,5 +230,6 @@ public void testAddAfterLastFieldRemoved() { .add(1, 1001, "id_bucket_8", "bucket[8]") .build(), table.spec()); Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); + Assert.assertEquals(1001, table.ops().current().lastAssignedPartitionId()); } } diff --git a/core/src/test/resources/TableMetadataV2MissingLastAssignedPartitionId.json b/core/src/test/resources/TableMetadataV2MissingLastAssignedPartitionId.json new file mode 100644 index 000000000000..3754354ede7b --- /dev/null +++ b/core/src/test/resources/TableMetadataV2MissingLastAssignedPartitionId.json @@ -0,0 +1,71 @@ +{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "schema": { + "type": "struct", + "fields": [ + { + "id": 1, + "name": "x", + "required": true, + "type": "long" + }, + { + "id": 2, + "name": "y", + "required": true, + "type": "long", + "doc": "comment" + }, + { + "id": 3, + "name": "z", + "required": true, + "type": "long" + } + ] + }, + "default-spec-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [ + { + "name": "x", + "transform": "identity", + "source-id": 1, + "field-id": 1000 + } + ] + } + ], + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + { + "transform": "identity", + "source-id": 2, + "direction": "asc", + "null-order": "nulls-first" + }, + { + "transform": "bucket[4]", + "source-id": 3, + "direction": "desc", + "null-order": "nulls-last" + } + ] + } + ], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [] +} \ No newline at end of file diff --git a/core/src/test/resources/TableMetadataV2MissingSortOrder.json b/core/src/test/resources/TableMetadataV2MissingSortOrder.json index 4a0c4d4d742d..f656e08ff5ad 100644 --- a/core/src/test/resources/TableMetadataV2MissingSortOrder.json +++ b/core/src/test/resources/TableMetadataV2MissingSortOrder.json @@ -43,6 +43,7 @@ ] } ], + "last-assigned-partition-id": 1000, "properties": {}, "current-snapshot-id": -1, "snapshots": [], diff --git a/core/src/test/resources/TableMetadataV2Valid.json b/core/src/test/resources/TableMetadataV2Valid.json index 3754354ede7b..aee3de6c360f 100644 --- a/core/src/test/resources/TableMetadataV2Valid.json +++ b/core/src/test/resources/TableMetadataV2Valid.json @@ -43,6 +43,7 @@ ] } ], + "last-assigned-partition-id": 1000, "default-sort-order-id": 3, "sort-orders": [ {