diff --git a/api/src/main/java/org/apache/iceberg/Schema.java b/api/src/main/java/org/apache/iceberg/Schema.java index 5eec30592a57..f753a2b44e07 100644 --- a/api/src/main/java/org/apache/iceberg/Schema.java +++ b/api/src/main/java/org/apache/iceberg/Schema.java @@ -40,12 +40,17 @@ /** * The schema of a data table. + *

+ * Schema ID will only be populated when reading from/writing to table metadata, + * otherwise it will be default to 0. */ public class Schema implements Serializable { private static final Joiner NEWLINE = Joiner.on('\n'); private static final String ALL_COLUMNS = "*"; + private static final int DEFAULT_SCHEMA_ID = 0; private final StructType struct; + private final int schemaId; private transient BiMap aliasToId = null; private transient Map idToField = null; private transient Map nameToId = null; @@ -54,6 +59,7 @@ public class Schema implements Serializable { private transient Map idToName = null; public Schema(List columns, Map aliases) { + this.schemaId = DEFAULT_SCHEMA_ID; this.struct = StructType.of(columns); this.aliasToId = aliases != null ? ImmutableBiMap.copyOf(aliases) : null; @@ -62,12 +68,21 @@ public Schema(List columns, Map aliases) { } public Schema(List columns) { + this(DEFAULT_SCHEMA_ID, columns); + } + + public Schema(int schemaId, List columns) { + this.schemaId = schemaId; this.struct = StructType.of(columns); lazyIdToName(); } public Schema(NestedField... columns) { - this(Arrays.asList(columns)); + this(DEFAULT_SCHEMA_ID, Arrays.asList(columns)); + } + + public Schema(int schemaId, NestedField... columns) { + this(schemaId, Arrays.asList(columns)); } private Map lazyIdToField() { @@ -105,6 +120,16 @@ private Map> lazyIdToAccessor() { return idToAccessor; } + /** + * Returns the schema ID for this schema. + *

+ * Note that schema ID will only be populated when reading from/writing to table metadata, + * otherwise it will be default to 0. + */ + public int schemaId() { + return this.schemaId; + } + /** * Returns an alias map for this schema, if set. *

diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index f875cf9c60bc..a6b8b62735f8 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -157,6 +157,21 @@ public static Schema assignFreshIds(Schema schema, NextID nextId) { .fields()); } + /** + * Assigns fresh ids from the {@link NextID nextId function} for all fields in a schema. + * + * @param schemaId an ID assigned to this schema + * @param schema a schema + * @param nextId an id assignment function + * @return a structurally identical schema with new ids assigned by the nextId function + */ + public static Schema assignFreshIds(int schemaId, Schema schema, NextID nextId) { + return new Schema(schemaId, TypeUtil + .visit(schema.asStruct(), new AssignFreshIds(nextId)) + .asNestedType() + .fields()); + } + /** * Assigns ids to match a given schema, and fresh ids from the {@link NextID nextId function} for all other fields. * diff --git a/api/src/test/java/org/apache/iceberg/TestHelpers.java b/api/src/test/java/org/apache/iceberg/TestHelpers.java index 091b032dadbb..8e3ebe261bce 100644 --- a/api/src/test/java/org/apache/iceberg/TestHelpers.java +++ b/api/src/test/java/org/apache/iceberg/TestHelpers.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.IntStream; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.BoundSetPredicate; import org.apache.iceberg.expressions.Expression; @@ -84,6 +85,23 @@ public static T roundTripSerialize(T type) throws IOException, ClassNotFound } } + public static void assertSameSchemaList(List list1, List list2) { + if (list1.size() != list2.size()) { + Assert.fail("Should have same number of schemas in both lists"); + } + + IntStream.range(0, list1.size()).forEach( + index -> { + Schema schema1 = list1.get(index); + Schema schema2 = list2.get(index); + Assert.assertEquals("Should have matching schema id", + schema1.schemaId(), schema2.schemaId()); + Assert.assertEquals("Should have matching schema struct", + schema1.asStruct(), schema2.asStruct()); + } + ); + } + private static class CheckReferencesBound extends ExpressionVisitors.ExpressionVisitor { private final String message; diff --git a/core/src/main/java/org/apache/iceberg/SchemaParser.java b/core/src/main/java/org/apache/iceberg/SchemaParser.java index 65fe5a93d41e..534e085e1287 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaParser.java +++ b/core/src/main/java/org/apache/iceberg/SchemaParser.java @@ -39,6 +39,7 @@ public class SchemaParser { private SchemaParser() { } + private static final String SCHEMA_ID = "schema-id"; private static final String TYPE = "type"; private static final String STRUCT = "struct"; private static final String LIST = "list"; @@ -57,10 +58,18 @@ private SchemaParser() { private static final String ELEMENT_REQUIRED = "element-required"; private static final String VALUE_REQUIRED = "value-required"; - static void toJson(Types.StructType struct, JsonGenerator generator) throws IOException { + private static void toJson(Types.StructType struct, JsonGenerator generator) throws IOException { + toJson(struct, null, generator); + } + + private static void toJson(Types.StructType struct, Integer schemaId, JsonGenerator generator) throws IOException { generator.writeStartObject(); generator.writeStringField(TYPE, STRUCT); + if (schemaId != null) { + generator.writeNumberField(SCHEMA_ID, schemaId); + } + generator.writeArrayFieldStart(FIELDS); for (Types.NestedField field : struct.fields()) { generator.writeStartObject(); @@ -135,7 +144,7 @@ static void toJson(Type type, JsonGenerator generator) throws IOException { } public static void toJson(Schema schema, JsonGenerator generator) throws IOException { - toJson(schema.asStruct(), generator); + toJson(schema.asStruct(), schema.schemaId(), generator); } public static String toJson(Schema schema) { @@ -237,7 +246,13 @@ public static Schema fromJson(JsonNode json) { Type type = typeFromJson(json); Preconditions.checkArgument(type.isNestedType() && type.asNestedType().isStructType(), "Cannot create schema, not a struct type: %s", type); - return new Schema(type.asNestedType().asStructType().fields()); + Integer schemaId = JsonUtil.getIntOrNull(SCHEMA_ID, json); + + if (schemaId == null) { + return new Schema(type.asNestedType().asStructType().fields()); + } else { + return new Schema(schemaId, type.asNestedType().asStructType().fields()); + } } private static final Cache SCHEMA_CACHE = Caffeine.newBuilder() diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index cf8e64b0203d..1e33c54b8ec6 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -52,6 +52,7 @@ public class TableMetadata implements Serializable { static final int SUPPORTED_TABLE_FORMAT_VERSION = 2; static final int INITIAL_SPEC_ID = 0; static final int INITIAL_SORT_ORDER_ID = 1; + static final int INITIAL_SCHEMA_ID = 0; private static final long ONE_MINUTE = TimeUnit.MINUTES.toMillis(1); @@ -90,7 +91,7 @@ static TableMetadata newTableMetadata(Schema schema, int formatVersion) { // reassign all column ids to ensure consistency AtomicInteger lastColumnId = new AtomicInteger(0); - Schema freshSchema = TypeUtil.assignFreshIds(schema, lastColumnId::incrementAndGet); + Schema freshSchema = TypeUtil.assignFreshIds(INITIAL_SCHEMA_ID, schema, lastColumnId::incrementAndGet); // rebuild the partition spec using the new column ids PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(freshSchema) @@ -116,8 +117,9 @@ static TableMetadata newTableMetadata(Schema schema, return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, INITIAL_SEQUENCE_NUMBER, System.currentTimeMillis(), - lastColumnId.get(), freshSchema, INITIAL_SPEC_ID, ImmutableList.of(freshSpec), - freshSpec.lastAssignedFieldId(), freshSortOrderId, ImmutableList.of(freshSortOrder), + lastColumnId.get(), freshSchema.schemaId(), ImmutableList.of(freshSchema), + freshSpec.specId(), ImmutableList.of(freshSpec), freshSpec.lastAssignedFieldId(), + freshSortOrderId, ImmutableList.of(freshSortOrder), ImmutableMap.copyOf(properties), -1, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()); } @@ -219,7 +221,8 @@ public String toString() { private final long lastSequenceNumber; private final long lastUpdatedMillis; private final int lastColumnId; - private final Schema schema; + private final int currentSchemaId; + private final List schemas; private final int defaultSpecId; private final List specs; private final int lastAssignedPartitionId; @@ -229,6 +232,7 @@ public String toString() { private final long currentSnapshotId; private final List snapshots; private final Map snapshotsById; + private final Map schemasById; private final Map specsById; private final Map sortOrdersById; private final List snapshotLog; @@ -242,7 +246,8 @@ public String toString() { long lastSequenceNumber, long lastUpdatedMillis, int lastColumnId, - Schema schema, + int currentSchemaId, + List schemas, int defaultSpecId, List specs, int lastAssignedPartitionId, @@ -270,7 +275,8 @@ public String toString() { this.lastSequenceNumber = lastSequenceNumber; this.lastUpdatedMillis = lastUpdatedMillis; this.lastColumnId = lastColumnId; - this.schema = schema; + this.currentSchemaId = currentSchemaId; + this.schemas = schemas; this.specs = specs; this.defaultSpecId = defaultSpecId; this.lastAssignedPartitionId = lastAssignedPartitionId; @@ -283,6 +289,7 @@ public String toString() { this.previousFiles = previousFiles; this.snapshotsById = indexAndValidateSnapshots(snapshots, lastSequenceNumber); + this.schemasById = indexSchemas(); this.specsById = indexSpecs(specs); this.sortOrdersById = indexSortOrders(sortOrders); @@ -359,7 +366,19 @@ public int lastColumnId() { } public Schema schema() { - return schema; + return schemasById.get(currentSchemaId); + } + + public List schemas() { + return schemas; + } + + public Map schemasById() { + return schemasById; + } + + public int currentSchemaId() { + return currentSchemaId; } public PartitionSpec spec() { @@ -451,9 +470,9 @@ public TableMetadata withUUID() { return this; } else { return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, - lastSequenceNumber, lastUpdatedMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId, - defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + lastSequenceNumber, lastUpdatedMillis, lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, + currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } } @@ -463,14 +482,29 @@ public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) { // rebuild all of the partition specs and sort orders for the new current schema List updatedSpecs = Lists.transform(specs, spec -> updateSpecSchema(newSchema, spec)); List updatedSortOrders = Lists.transform(sortOrders, order -> updateSortOrderSchema(newSchema, order)); + + int newSchemaId = reuseOrCreateNewSchemaId(newSchema); + if (currentSchemaId == newSchemaId && newLastColumnId == lastColumnId) { + // the new spec and last column Id is already current and no change is needed + return this; + } + + ImmutableList.Builder builder = ImmutableList.builder().addAll(schemas); + if (!schemasById.containsKey(newSchemaId)) { + builder.add(new Schema(newSchemaId, newSchema.columns())); + } + return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, newSchema, defaultSpecId, updatedSpecs, - lastAssignedPartitionId, defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); + lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, + newSchemaId, builder.build(), defaultSpecId, updatedSpecs, lastAssignedPartitionId, + defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, snapshots, snapshotLog, + addPreviousFile(file, lastUpdatedMillis)); } // The caller is responsible to pass a newPartitionSpec with correct partition field IDs public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { + Schema schema = schema(); + PartitionSpec.checkCompatibility(newPartitionSpec, schema); ValidationException.check(formatVersion > 1 || PartitionSpec.hasSequentialIds(newPartitionSpec), "Spec does not use sequential IDs that are required in v1: %s", newPartitionSpec); @@ -499,13 +533,14 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { } return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, newDefaultSpecId, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, newDefaultSpecId, builder.build(), Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId()), defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } public TableMetadata replaceSortOrder(SortOrder newOrder) { + Schema schema = schema(); SortOrder.checkCompatibility(newOrder, schema); // determine the next order id @@ -537,7 +572,7 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { } return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -553,9 +588,10 @@ public TableMetadata addStagedSnapshot(Snapshot snapshot) { .build(); return new TableMetadata(null, formatVersion, uuid, location, - snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - newSnapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); + snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, + currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, + defaultSortOrderId, sortOrders, properties, currentSnapshotId, newSnapshots, snapshotLog, + addPreviousFile(file, lastUpdatedMillis)); } public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { @@ -578,9 +614,10 @@ public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { .build(); return new TableMetadata(null, formatVersion, uuid, location, - snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, schema, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), - newSnapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); + snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, + currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, + defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), newSnapshots, newSnapshotLog, + addPreviousFile(file, lastUpdatedMillis)); } public TableMetadata removeSnapshotsIf(Predicate removeIf) { @@ -610,7 +647,7 @@ public TableMetadata removeSnapshotsIf(Predicate removeIf) { } return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, ImmutableList.copyOf(newSnapshotLog), addPreviousFile(file, lastUpdatedMillis)); } @@ -634,15 +671,15 @@ private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { .build(); return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, nowMillis, lastColumnId, schema, defaultSpecId, specs, lastAssignedPartitionId, - defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, newSnapshotLog, - addPreviousFile(file, lastUpdatedMillis)); + lastSequenceNumber, nowMillis, lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, + newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } public TableMetadata replaceProperties(Map newProperties) { ValidationException.check(newProperties != null, "Cannot set properties to null"); return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis, newProperties)); } @@ -661,7 +698,7 @@ public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { "Cannot set invalid snapshot log: latest entry is not the current snapshot"); return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, newSnapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -674,7 +711,7 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update "Spec does not use sequential IDs that are required in v1: %s", updatedPartitionSpec); AtomicInteger newLastColumnId = new AtomicInteger(lastColumnId); - Schema freshSchema = TypeUtil.assignFreshIds(updatedSchema, schema, newLastColumnId::incrementAndGet); + Schema freshSchema = TypeUtil.assignFreshIds(updatedSchema, schema(), newLastColumnId::incrementAndGet); // determine the next spec id OptionalInt maxSpecId = specs.stream().mapToInt(PartitionSpec::specId).max(); @@ -718,8 +755,16 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update newProperties.putAll(this.properties); newProperties.putAll(updatedProperties); + // determine the next schema id + int freshSchemaId = reuseOrCreateNewSchemaId(freshSchema); + ImmutableList.Builder schemasBuilder = ImmutableList.builder().addAll(schemas); + + if (!schemasById.containsKey(freshSchemaId)) { + schemasBuilder.add(new Schema(freshSchemaId, freshSchema.columns())); + } + return new TableMetadata(null, formatVersion, uuid, newLocation, - lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchema, + lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchemaId, schemasBuilder.build(), specId, specListBuilder.build(), Math.max(lastAssignedPartitionId, freshSpec.lastAssignedFieldId()), orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties), -1, snapshots, ImmutableList.of(), addPreviousFile(file, lastUpdatedMillis, newProperties)); @@ -727,7 +772,7 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update public TableMetadata updateLocation(String newLocation) { return new TableMetadata(null, formatVersion, uuid, newLocation, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -744,7 +789,7 @@ public TableMetadata upgradeToFormatVersion(int newFormatVersion) { } return new TableMetadata(null, newFormatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schema, defaultSpecId, specs, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, snapshots, snapshotLog, addPreviousFile(file, lastUpdatedMillis)); } @@ -843,6 +888,14 @@ private static Map indexAndValidateSnapshots(List snap return builder.build(); } + private Map indexSchemas() { + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Schema schema : schemas) { + builder.put(schema.schemaId(), schema); + } + return builder.build(); + } + private static Map indexSpecs(List specs) { ImmutableMap.Builder builder = ImmutableMap.builder(); for (PartitionSpec spec : specs) { @@ -858,4 +911,18 @@ private static Map indexSortOrders(List sortOrder } return builder.build(); } + + private int reuseOrCreateNewSchemaId(Schema newSchema) { + // if the schema already exists, use its id; otherwise use the highest id + 1 + int newSchemaId = currentSchemaId; + for (Schema schema : schemas) { + if (schema.asStruct().equals(newSchema.asStruct())) { + newSchemaId = schema.schemaId(); + break; + } else if (schema.schemaId() >= newSchemaId) { + newSchemaId = schema.schemaId() + 1; + } + } + return newSchemaId; + } } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index 1cb997554827..69dae5c16eba 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -88,6 +88,8 @@ private TableMetadataParser() { static final String LAST_UPDATED_MILLIS = "last-updated-ms"; static final String LAST_COLUMN_ID = "last-column-id"; static final String SCHEMA = "schema"; + static final String SCHEMAS = "schemas"; + static final String CURRENT_SCHEMA_ID = "current-schema-id"; static final String PARTITION_SPEC = "partition-spec"; static final String PARTITION_SPECS = "partition-specs"; static final String DEFAULT_SPEC_ID = "default-spec-id"; @@ -162,8 +164,20 @@ private static void toJson(TableMetadata metadata, JsonGenerator generator) thro generator.writeNumberField(LAST_UPDATED_MILLIS, metadata.lastUpdatedMillis()); generator.writeNumberField(LAST_COLUMN_ID, metadata.lastColumnId()); - generator.writeFieldName(SCHEMA); - SchemaParser.toJson(metadata.schema(), generator); + // for older readers, continue writing the current schema as "schema". + // this is only needed for v1 because support for schemas and current-schema-id is required in v2 and later. + if (metadata.formatVersion() == 1) { + generator.writeFieldName(SCHEMA); + SchemaParser.toJson(metadata.schema(), generator); + } + + // write the current schema ID and schema list + generator.writeNumberField(CURRENT_SCHEMA_ID, metadata.currentSchemaId()); + generator.writeArrayFieldStart(SCHEMAS); + for (Schema schema : metadata.schemas()) { + SchemaParser.toJson(schema, generator); + } + generator.writeEndArray(); // for older readers, continue writing the default spec as "partition-spec" if (metadata.formatVersion() == 1) { @@ -245,6 +259,7 @@ public static TableMetadata read(FileIO io, InputFile file) { } } + @SuppressWarnings("checkstyle:CyclomaticComplexity") static TableMetadata fromJson(FileIO io, InputFile file, JsonNode node) { Preconditions.checkArgument(node.isObject(), "Cannot parse metadata from a non-object: %s", node); @@ -262,7 +277,41 @@ static TableMetadata fromJson(FileIO io, InputFile file, JsonNode node) { lastSequenceNumber = TableMetadata.INITIAL_SEQUENCE_NUMBER; } int lastAssignedColumnId = JsonUtil.getInt(LAST_COLUMN_ID, node); - Schema schema = SchemaParser.fromJson(node.get(SCHEMA)); + + List schemas; + int currentSchemaId; + Schema schema = null; + + JsonNode schemaArray = node.get(SCHEMAS); + if (schemaArray != null) { + Preconditions.checkArgument(schemaArray.isArray(), + "Cannot parse schemas from non-array: %s", schemaArray); + // current schema ID is required when the schema array is present + currentSchemaId = JsonUtil.getInt(CURRENT_SCHEMA_ID, node); + + // parse the schema array + ImmutableList.Builder builder = ImmutableList.builder(); + for (JsonNode schemaNode : schemaArray) { + Schema current = SchemaParser.fromJson(schemaNode); + if (current.schemaId() == currentSchemaId) { + schema = current; + } + builder.add(current); + } + + Preconditions.checkArgument(schema != null, + "Cannot find schema with %s=%s from %s", CURRENT_SCHEMA_ID, currentSchemaId, SCHEMAS); + + schemas = builder.build(); + + } else { + Preconditions.checkArgument(formatVersion == 1, + "%s must exist in format v%s", SCHEMAS, formatVersion); + + schema = SchemaParser.fromJson(node.get(SCHEMA)); + currentSchemaId = schema.schemaId(); + schemas = ImmutableList.of(schema); + } JsonNode specArray = node.get(PARTITION_SPECS); List specs; @@ -351,7 +400,7 @@ static TableMetadata fromJson(FileIO io, InputFile file, JsonNode node) { } return new TableMetadata(file, formatVersion, uuid, location, - lastSequenceNumber, lastUpdatedMillis, lastAssignedColumnId, schema, defaultSpecId, specs, + lastSequenceNumber, lastUpdatedMillis, lastAssignedColumnId, currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentVersionId, snapshots, entries.build(), metadataEntries.build()); } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 62b35f84e038..42c46076b134 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -59,11 +59,12 @@ import static org.apache.iceberg.TableMetadataParser.PROPERTIES; import static org.apache.iceberg.TableMetadataParser.SCHEMA; import static org.apache.iceberg.TableMetadataParser.SNAPSHOTS; +import static org.apache.iceberg.TestHelpers.assertSameSchemaList; public class TestTableMetadata { private static final String TEST_LOCATION = "s3://bucket/test/location"; - private static final Schema TEST_SCHEMA = new Schema( + private static final Schema TEST_SCHEMA = new Schema(7, Types.NestedField.required(1, "x", Types.LongType.get()), Types.NestedField.required(2, "y", Types.LongType.get(), "comment"), Types.NestedField.required(3, "z", Types.LongType.get()) @@ -100,8 +101,14 @@ public void testJsonConversion() throws Exception { .add(new SnapshotLogEntry(currentSnapshot.timestampMillis(), currentSnapshot.snapshotId())) .build(); + Schema schema = new Schema(6, + Types.NestedField.required(10, "x", Types.StringType.get())); + TableMetadata expected = new TableMetadata(null, 2, UUID.randomUUID().toString(), TEST_LOCATION, - SEQ_NO, System.currentTimeMillis(), 3, TEST_SCHEMA, 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), + SEQ_NO, System.currentTimeMillis(), 3, + 7, ImmutableList.of(TEST_SCHEMA, schema), + 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), + 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), snapshotLog, ImmutableList.of()); @@ -119,8 +126,9 @@ public void testJsonConversion() throws Exception { expected.lastSequenceNumber(), metadata.lastSequenceNumber()); Assert.assertEquals("Last column ID should match", expected.lastColumnId(), metadata.lastColumnId()); - Assert.assertEquals("Schema should match", - expected.schema().asStruct(), metadata.schema().asStruct()); + Assert.assertEquals("Current schema id should match", + expected.currentSchemaId(), metadata.currentSchemaId()); + assertSameSchemaList(expected.schemas(), metadata.schemas()); Assert.assertEquals("Partition spec should match", expected.spec().toString(), metadata.spec().toString()); Assert.assertEquals("Default spec ID should match", @@ -150,6 +158,7 @@ public void testJsonConversion() throws Exception { public void testBackwardCompat() throws Exception { PartitionSpec spec = PartitionSpec.builderFor(TEST_SCHEMA).identity("x").withSpecId(6).build(); SortOrder sortOrder = SortOrder.unsorted(); + Schema schema = new Schema(TableMetadata.INITIAL_SCHEMA_ID, TEST_SCHEMA.columns()); long previousSnapshotId = System.currentTimeMillis() - new Random(1234).nextInt(3600); Snapshot previousSnapshot = new BaseSnapshot( @@ -161,11 +170,12 @@ public void testBackwardCompat() throws Exception { new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), spec.specId()))); TableMetadata expected = new TableMetadata(null, 1, null, TEST_LOCATION, - 0, System.currentTimeMillis(), 3, TEST_SCHEMA, 6, ImmutableList.of(spec), spec.lastAssignedFieldId(), + 0, System.currentTimeMillis(), 3, TableMetadata.INITIAL_SCHEMA_ID, + ImmutableList.of(schema), 6, ImmutableList.of(spec), spec.lastAssignedFieldId(), TableMetadata.INITIAL_SORT_ORDER_ID, ImmutableList.of(sortOrder), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), ImmutableList.of(), ImmutableList.of()); - String asJson = toJsonWithoutSpecList(expected); + String asJson = toJsonWithoutSpecAndSchemaList(expected); TableMetadata metadata = TableMetadataParser .fromJson(ops.io(), null, JsonUtil.mapper().readValue(asJson, JsonNode.class)); @@ -178,8 +188,12 @@ public void testBackwardCompat() throws Exception { expected.lastSequenceNumber(), metadata.lastSequenceNumber()); Assert.assertEquals("Last column ID should match", expected.lastColumnId(), metadata.lastColumnId()); - Assert.assertEquals("Schema should match", - expected.schema().asStruct(), metadata.schema().asStruct()); + Assert.assertEquals("Current schema ID should be default to TableMetadata.INITIAL_SCHEMA_ID", + TableMetadata.INITIAL_SCHEMA_ID, metadata.currentSchemaId()); + Assert.assertEquals("Schemas size should match", + 1, metadata.schemas().size()); + Assert.assertEquals("Schemas should contain the schema", + metadata.schemas().get(0).asStruct(), schema.asStruct()); Assert.assertEquals("Partition spec should be the default", expected.spec().toString(), metadata.spec().toString()); Assert.assertEquals("Default spec ID should default to TableMetadata.INITIAL_SPEC_ID", @@ -211,7 +225,7 @@ public void testBackwardCompat() throws Exception { expected.previousFiles(), metadata.previousFiles()); } - public static String toJsonWithoutSpecList(TableMetadata metadata) { + private static String toJsonWithoutSpecAndSchemaList(TableMetadata metadata) { StringWriter writer = new StringWriter(); try { JsonGenerator generator = JsonUtil.factory().createGenerator(writer); @@ -223,6 +237,7 @@ public static String toJsonWithoutSpecList(TableMetadata metadata) { generator.writeNumberField(LAST_UPDATED_MILLIS, metadata.lastUpdatedMillis()); generator.writeNumberField(LAST_COLUMN_ID, metadata.lastColumnId()); + // mimic an old writer by writing only schema and not the current ID or schema list generator.writeFieldName(SCHEMA); SchemaParser.toJson(metadata.schema(), generator); @@ -273,7 +288,8 @@ public void testJsonWithPreviousMetadataLog() throws Exception { "/tmp/000001-" + UUID.randomUUID().toString() + ".metadata.json")); TableMetadata base = new TableMetadata(null, 1, UUID.randomUUID().toString(), TEST_LOCATION, - 0, System.currentTimeMillis(), 3, TEST_SCHEMA, 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), + 0, System.currentTimeMillis(), 3, + 7, ImmutableList.of(TEST_SCHEMA), 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, ImmutableList.copyOf(previousMetadataLog)); @@ -308,8 +324,8 @@ public void testAddPreviousMetadataRemoveNone() { "/tmp/000003-" + UUID.randomUUID().toString() + ".metadata.json"); TableMetadata base = new TableMetadata(localInput(latestPreviousMetadata.file()), 1, UUID.randomUUID().toString(), - TEST_LOCATION, 0, currentTimestamp - 80, 3, TEST_SCHEMA, 5, ImmutableList.of(SPEC_5), - SPEC_5.lastAssignedFieldId(), + TEST_LOCATION, 0, currentTimestamp - 80, 3, + 7, ImmutableList.of(TEST_SCHEMA), 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, ImmutableList.copyOf(previousMetadataLog)); @@ -354,7 +370,8 @@ public void testAddPreviousMetadataRemoveOne() { "/tmp/000006-" + UUID.randomUUID().toString() + ".metadata.json"); TableMetadata base = new TableMetadata(localInput(latestPreviousMetadata.file()), 1, UUID.randomUUID().toString(), - TEST_LOCATION, 0, currentTimestamp - 50, 3, TEST_SCHEMA, 5, + TEST_LOCATION, 0, currentTimestamp - 50, 3, + 7, ImmutableList.of(TEST_SCHEMA), 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, @@ -405,9 +422,10 @@ public void testAddPreviousMetadataRemoveMultiple() { "/tmp/000006-" + UUID.randomUUID().toString() + ".metadata.json"); TableMetadata base = new TableMetadata(localInput(latestPreviousMetadata.file()), 1, UUID.randomUUID().toString(), - TEST_LOCATION, 0, currentTimestamp - 50, 3, TEST_SCHEMA, 2, - ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), TableMetadata.INITIAL_SORT_ORDER_ID, - ImmutableList.of(SortOrder.unsorted()), ImmutableMap.of("property", "value"), currentSnapshotId, + TEST_LOCATION, 0, currentTimestamp - 50, 3, 7, ImmutableList.of(TEST_SCHEMA), 2, + ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), + TableMetadata.INITIAL_SORT_ORDER_ID, ImmutableList.of(SortOrder.unsorted()), + ImmutableMap.of("property", "value"), currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, ImmutableList.copyOf(previousMetadataLog)); @@ -432,8 +450,9 @@ public void testV2UUIDValidation() { AssertHelpers.assertThrows("Should reject v2 metadata without a UUID", IllegalArgumentException.class, "UUID is required in format v2", () -> new TableMetadata(null, 2, null, TEST_LOCATION, SEQ_NO, System.currentTimeMillis(), - LAST_ASSIGNED_COLUMN_ID, TEST_SCHEMA, SPEC_5.specId(), ImmutableList.of(SPEC_5), - SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, + LAST_ASSIGNED_COLUMN_ID, 7, ImmutableList.of(TEST_SCHEMA), + SPEC_5.specId(), ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), + 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()) ); } @@ -444,7 +463,8 @@ public void testVersionValidation() { AssertHelpers.assertThrows("Should reject unsupported metadata", IllegalArgumentException.class, "Unsupported format version: v" + unsupportedVersion, () -> new TableMetadata(null, unsupportedVersion, null, TEST_LOCATION, SEQ_NO, - System.currentTimeMillis(), LAST_ASSIGNED_COLUMN_ID, TEST_SCHEMA, SPEC_5.specId(), ImmutableList.of(SPEC_5), + System.currentTimeMillis(), LAST_ASSIGNED_COLUMN_ID, + 7, ImmutableList.of(TEST_SCHEMA), SPEC_5.specId(), ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, ImmutableList.of(), ImmutableList.of(), ImmutableList.of()) ); @@ -501,6 +521,26 @@ public void testParserV2SortOrderValidation() throws Exception { ); } + @Test + public void testParserV2CurrentSchemaIdValidation() throws Exception { + String unsupported = readTableMetadataInputFile("TableMetadataV2CurrentSchemaNotFound.json"); + AssertHelpers.assertThrows("Should reject v2 metadata without valid schema id", + IllegalArgumentException.class, "Cannot find schema with current-schema-id=2 from schemas", + () -> TableMetadataParser.fromJson( + ops.io(), null, JsonUtil.mapper().readValue(unsupported, JsonNode.class)) + ); + } + + @Test + public void testParserV2SchemasValidation() throws Exception { + String unsupported = readTableMetadataInputFile("TableMetadataV2MissingSchemas.json"); + AssertHelpers.assertThrows("Should reject v2 metadata without schemas", + IllegalArgumentException.class, "schemas must exist in format v2", + () -> TableMetadataParser.fromJson( + ops.io(), null, JsonUtil.mapper().readValue(unsupported, JsonNode.class)) + ); + } + private String readTableMetadataInputFile(String fileName) throws Exception { Path path = Paths.get(getClass().getClassLoader().getResource(fileName).toURI()); return String.join("", java.nio.file.Files.readAllLines(path)); @@ -599,4 +639,81 @@ public void testUpdateSortOrder() { Assert.assertEquals("Should be nulls first", NullOrder.NULLS_FIRST, sortedByX.sortOrder().fields().get(0).nullOrder()); } + + @Test + public void testUpdateSchema() { + Schema schema = new Schema(0, + Types.NestedField.required(1, "y", Types.LongType.get(), "comment") + ); + TableMetadata freshTable = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), null, ImmutableMap.of()); + Assert.assertEquals("Should use TableMetadata.INITIAL_SCHEMA_ID for current schema id", + TableMetadata.INITIAL_SCHEMA_ID, freshTable.currentSchemaId()); + assertSameSchemaList(ImmutableList.of(schema), freshTable.schemas()); + Assert.assertEquals("Should have expected schema upon return", + schema.asStruct(), freshTable.schema().asStruct()); + Assert.assertEquals("Should return expected last column id", 1, freshTable.lastColumnId()); + + // update schema + Schema schema2 = new Schema( + Types.NestedField.required(1, "y", Types.LongType.get(), "comment"), + Types.NestedField.required(2, "x", Types.StringType.get()) + ); + TableMetadata twoSchemasTable = freshTable.updateSchema(schema2, 2); + Assert.assertEquals("Should have current schema id as 1", + 1, twoSchemasTable.currentSchemaId()); + assertSameSchemaList(ImmutableList.of(schema, new Schema(1, schema2.columns())), + twoSchemasTable.schemas()); + Assert.assertEquals("Should have expected schema upon return", + schema2.asStruct(), twoSchemasTable.schema().asStruct()); + Assert.assertEquals("Should return expected last column id", 2, twoSchemasTable.lastColumnId()); + + // update schema with the the same schema and last column ID as current shouldn't cause change + Schema sameSchema2 = new Schema( + Types.NestedField.required(1, "y", Types.LongType.get(), "comment"), + Types.NestedField.required(2, "x", Types.StringType.get()) + ); + TableMetadata sameSchemaTable = twoSchemasTable.updateSchema(sameSchema2, 2); + Assert.assertEquals("Should return same table metadata", + twoSchemasTable, sameSchemaTable); + + // update schema with the the same schema and different last column ID as current should create a new table + TableMetadata differentColumnIdTable = sameSchemaTable.updateSchema(sameSchema2, 3); + Assert.assertEquals("Should have current schema id as 1", + 1, differentColumnIdTable.currentSchemaId()); + assertSameSchemaList(ImmutableList.of(schema, new Schema(1, schema2.columns())), + differentColumnIdTable.schemas()); + Assert.assertEquals("Should have expected schema upon return", + schema2.asStruct(), differentColumnIdTable.schema().asStruct()); + Assert.assertEquals("Should return expected last column id", + 3, differentColumnIdTable.lastColumnId()); + + // update schema with old schema does not change schemas + TableMetadata revertSchemaTable = differentColumnIdTable.updateSchema(schema, 3); + Assert.assertEquals("Should have current schema id as 0", + 0, revertSchemaTable.currentSchemaId()); + assertSameSchemaList(ImmutableList.of(schema, new Schema(1, schema2.columns())), + revertSchemaTable.schemas()); + Assert.assertEquals("Should have expected schema upon return", + schema.asStruct(), revertSchemaTable.schema().asStruct()); + Assert.assertEquals("Should return expected last column id", + 3, revertSchemaTable.lastColumnId()); + + // create new schema will use the largest schema id + 1 + Schema schema3 = new Schema( + Types.NestedField.required(2, "y", Types.LongType.get(), "comment"), + Types.NestedField.required(4, "x", Types.StringType.get()), + Types.NestedField.required(6, "z", Types.IntegerType.get()) + ); + TableMetadata threeSchemaTable = revertSchemaTable.updateSchema(schema3, 3); + Assert.assertEquals("Should have current schema id as 2", + 2, threeSchemaTable.currentSchemaId()); + assertSameSchemaList(ImmutableList.of(schema, + new Schema(1, schema2.columns()), + new Schema(2, schema3.columns())), threeSchemaTable.schemas()); + Assert.assertEquals("Should have expected schema upon return", + schema3.asStruct(), threeSchemaTable.schema().asStruct()); + Assert.assertEquals("Should return expected last column id", + 3, threeSchemaTable.lastColumnId()); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadataSerialization.java b/core/src/test/java/org/apache/iceberg/TestTableMetadataSerialization.java index f2129fff6116..e140b469d0a7 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadataSerialization.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadataSerialization.java @@ -29,6 +29,8 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.apache.iceberg.TestHelpers.assertSameSchemaList; + @RunWith(Parameterized.class) public class TestTableMetadataSerialization extends TableTestBase { @Parameterized.Parameters(name = "formatVersion = {0}") @@ -68,6 +70,8 @@ public void testSerialization() throws Exception { Assert.assertEquals("Last updated should match", meta.lastUpdatedMillis(), result.lastUpdatedMillis()); Assert.assertEquals("Last column id", meta.lastColumnId(), result.lastColumnId()); Assert.assertEquals("Schema should match", meta.schema().asStruct(), result.schema().asStruct()); + assertSameSchemaList(meta.schemas(), result.schemas()); + Assert.assertEquals("Current schema id should match", meta.currentSchemaId(), result.currentSchemaId()); Assert.assertEquals("Spec should match", meta.defaultSpecId(), result.defaultSpecId()); Assert.assertEquals("Spec list should match", meta.specs(), result.specs()); Assert.assertEquals("Properties should match", meta.properties(), result.properties()); diff --git a/core/src/test/resources/TableMetadataV2CurrentSchemaNotFound.json b/core/src/test/resources/TableMetadataV2CurrentSchemaNotFound.json new file mode 100644 index 000000000000..8c42ca7f815a --- /dev/null +++ b/core/src/test/resources/TableMetadataV2CurrentSchemaNotFound.json @@ -0,0 +1,87 @@ +{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 2, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "x", + "required": true, + "type": "long" + } + ] + }, + { + "type": "struct", + "schema-id": 1, + "fields": [ + { + "id": 1, + "name": "x", + "required": true, + "type": "long" + }, + { + "id": 2, + "name": "y", + "required": true, + "type": "long", + "doc": "comment" + }, + { + "id": 3, + "name": "z", + "required": true, + "type": "long" + } + ] + } + ], + "default-spec-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [ + { + "name": "x", + "transform": "identity", + "source-id": 1, + "field-id": 1000 + } + ] + } + ], + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + { + "transform": "identity", + "source-id": 2, + "direction": "asc", + "null-order": "nulls-first" + }, + { + "transform": "bucket[4]", + "source-id": 3, + "direction": "desc", + "null-order": "nulls-last" + } + ] + } + ], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [] +} \ No newline at end of file diff --git a/core/src/test/resources/TableMetadataV2MissingLastPartitionId.json b/core/src/test/resources/TableMetadataV2MissingLastPartitionId.json index 3754354ede7b..31c2b4cafc0c 100644 --- a/core/src/test/resources/TableMetadataV2MissingLastPartitionId.json +++ b/core/src/test/resources/TableMetadataV2MissingLastPartitionId.json @@ -5,8 +5,10 @@ "last-sequence-number": 34, "last-updated-ms": 1602638573590, "last-column-id": 3, - "schema": { + "current-schema-id": 0, + "schemas": [{ "type": "struct", + "schema-id": 0, "fields": [ { "id": 1, @@ -28,7 +30,7 @@ "type": "long" } ] - }, + }], "default-spec-id": 0, "partition-specs": [ { diff --git a/core/src/test/resources/TableMetadataV2MissingPartitionSpecs.json b/core/src/test/resources/TableMetadataV2MissingPartitionSpecs.json index 934c7ef3a548..3ab0a7a1e20d 100644 --- a/core/src/test/resources/TableMetadataV2MissingPartitionSpecs.json +++ b/core/src/test/resources/TableMetadataV2MissingPartitionSpecs.json @@ -5,8 +5,10 @@ "last-sequence-number": 34, "last-updated-ms": 1602638573590, "last-column-id": 3, - "schema": { + "current-schema-id": 0, + "schemas": [{ "type": "struct", + "schema-id": 0, "fields": [ { "id": 1, @@ -28,7 +30,7 @@ "type": "long" } ] - }, + }], "partition-spec": [ { "name": "x", diff --git a/core/src/test/resources/TableMetadataV2MissingSchemas.json b/core/src/test/resources/TableMetadataV2MissingSchemas.json new file mode 100644 index 000000000000..3754354ede7b --- /dev/null +++ b/core/src/test/resources/TableMetadataV2MissingSchemas.json @@ -0,0 +1,71 @@ +{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "schema": { + "type": "struct", + "fields": [ + { + "id": 1, + "name": "x", + "required": true, + "type": "long" + }, + { + "id": 2, + "name": "y", + "required": true, + "type": "long", + "doc": "comment" + }, + { + "id": 3, + "name": "z", + "required": true, + "type": "long" + } + ] + }, + "default-spec-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [ + { + "name": "x", + "transform": "identity", + "source-id": 1, + "field-id": 1000 + } + ] + } + ], + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + { + "transform": "identity", + "source-id": 2, + "direction": "asc", + "null-order": "nulls-first" + }, + { + "transform": "bucket[4]", + "source-id": 3, + "direction": "desc", + "null-order": "nulls-last" + } + ] + } + ], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [] +} \ No newline at end of file diff --git a/core/src/test/resources/TableMetadataV2MissingSortOrder.json b/core/src/test/resources/TableMetadataV2MissingSortOrder.json index 93e6f879d4f0..fbbcf415d264 100644 --- a/core/src/test/resources/TableMetadataV2MissingSortOrder.json +++ b/core/src/test/resources/TableMetadataV2MissingSortOrder.json @@ -5,8 +5,10 @@ "last-sequence-number": 34, "last-updated-ms": 1602638573590, "last-column-id": 3, - "schema": { + "current-schema-id": 0, + "schemas": [{ "type": "struct", + "schema-id": 0, "fields": [ { "id": 1, @@ -28,7 +30,7 @@ "type": "long" } ] - }, + }], "default-spec-id": 0, "partition-specs": [ { diff --git a/core/src/test/resources/TableMetadataV2Valid.json b/core/src/test/resources/TableMetadataV2Valid.json index fabd9726561c..cf492f568ff3 100644 --- a/core/src/test/resources/TableMetadataV2Valid.json +++ b/core/src/test/resources/TableMetadataV2Valid.json @@ -5,30 +5,46 @@ "last-sequence-number": 34, "last-updated-ms": 1602638573590, "last-column-id": 3, - "schema": { - "type": "struct", - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - }, - { - "id": 2, - "name": "y", - "required": true, - "type": "long", - "doc": "comment" - }, - { - "id": 3, - "name": "z", - "required": true, - "type": "long" - } - ] - }, + "current-schema-id": 1, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "x", + "required": true, + "type": "long" + } + ] + }, + { + "type": "struct", + "schema-id": 1, + "fields": [ + { + "id": 1, + "name": "x", + "required": true, + "type": "long" + }, + { + "id": 2, + "name": "y", + "required": true, + "type": "long", + "doc": "comment" + }, + { + "id": 3, + "name": "z", + "required": true, + "type": "long" + } + ] + } + ], "default-spec-id": 0, "partition-specs": [ {