diff --git a/core/src/main/java/org/apache/iceberg/MetadataUpdate.java b/core/src/main/java/org/apache/iceberg/MetadataUpdate.java index 819974dfe476..2cf16bca6c32 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataUpdate.java +++ b/core/src/main/java/org/apache/iceberg/MetadataUpdate.java @@ -23,10 +23,20 @@ import java.util.Map; import java.util.Set; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.view.ViewMetadata; +import org.apache.iceberg.view.ViewVersion; -/** Represents a change to table metadata. */ +/** Represents a change to table or view metadata. */ public interface MetadataUpdate extends Serializable { - void applyTo(TableMetadata.Builder metadataBuilder); + default void applyTo(TableMetadata.Builder metadataBuilder) { + throw new UnsupportedOperationException( + String.format("Cannot apply update %s to a table", this.getClass().getSimpleName())); + } + + default void applyTo(ViewMetadata.Builder viewMetadataBuilder) { + throw new UnsupportedOperationException( + String.format("Cannot apply update %s to a view", this.getClass().getSimpleName())); + } class AssignUUID implements MetadataUpdate { private final String uuid; @@ -60,6 +70,11 @@ public int formatVersion() { public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.upgradeFormatVersion(formatVersion); } + + @Override + public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { + viewMetadataBuilder.upgradeFormatVersion(formatVersion); + } } class AddSchema implements MetadataUpdate { @@ -83,6 +98,11 @@ public int lastColumnId() { public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.addSchema(schema, lastColumnId); } + + @Override + public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { + viewMetadataBuilder.addSchema(schema); + } } class SetCurrentSchema implements MetadataUpdate { @@ -343,6 +363,11 @@ public Map updated() { public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setProperties(updated); } + + @Override + public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { + viewMetadataBuilder.setProperties(updated); + } } class RemoveProperties implements MetadataUpdate { @@ -360,6 +385,11 @@ public Set removed() { public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.removeProperties(removed); } + + @Override + public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { + viewMetadataBuilder.removeProperties(removed); + } } class SetLocation implements MetadataUpdate { @@ -377,5 +407,44 @@ public String location() { public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.setLocation(location); } + + @Override + public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { + viewMetadataBuilder.setLocation(location); + } + } + + class AddViewVersion implements MetadataUpdate { + private final ViewVersion viewVersion; + + public AddViewVersion(ViewVersion viewVersion) { + this.viewVersion = viewVersion; + } + + public ViewVersion viewVersion() { + return viewVersion; + } + + @Override + public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { + viewMetadataBuilder.addVersion(viewVersion); + } + } + + class SetCurrentViewVersion implements MetadataUpdate { + private final int versionId; + + public SetCurrentViewVersion(int versionId) { + this.versionId = versionId; + } + + public int versionId() { + return versionId; + } + + @Override + public void applyTo(ViewMetadata.Builder viewMetadataBuilder) { + viewMetadataBuilder.setCurrentVersionId(versionId); + } } } diff --git a/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java b/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java index 8c9b1b3f4eaa..e88f749c80f0 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java +++ b/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java @@ -29,6 +29,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.JsonUtil; +import org.apache.iceberg.view.ViewVersionParser; public class MetadataUpdateParser { @@ -54,6 +55,8 @@ private MetadataUpdateParser() {} static final String SET_LOCATION = "set-location"; static final String SET_STATISTICS = "set-statistics"; static final String REMOVE_STATISTICS = "remove-statistics"; + static final String ADD_VIEW_VERSION = "add-view-version"; + static final String SET_CURRENT_VIEW_VERSION = "set-current-view-version"; // AssignUUID private static final String UUID = "uuid"; @@ -112,6 +115,12 @@ private MetadataUpdateParser() {} // SetLocation private static final String LOCATION = "location"; + // AddViewVersion + private static final String VIEW_VERSION = "view-version"; + + // SetCurrentViewVersion + private static final String VIEW_VERSION_ID = "view-version-id"; + private static final Map, String> ACTIONS = ImmutableMap., String>builder() .put(MetadataUpdate.AssignUUID.class, ASSIGN_UUID) @@ -131,6 +140,8 @@ private MetadataUpdateParser() {} .put(MetadataUpdate.SetProperties.class, SET_PROPERTIES) .put(MetadataUpdate.RemoveProperties.class, REMOVE_PROPERTIES) .put(MetadataUpdate.SetLocation.class, SET_LOCATION) + .put(MetadataUpdate.AddViewVersion.class, ADD_VIEW_VERSION) + .put(MetadataUpdate.SetCurrentViewVersion.class, SET_CURRENT_VIEW_VERSION) .buildOrThrow(); public static String toJson(MetadataUpdate metadataUpdate) { @@ -208,6 +219,13 @@ public static void toJson(MetadataUpdate metadataUpdate, JsonGenerator generator case SET_LOCATION: writeSetLocation((MetadataUpdate.SetLocation) metadataUpdate, generator); break; + case ADD_VIEW_VERSION: + writeAddViewVersion((MetadataUpdate.AddViewVersion) metadataUpdate, generator); + break; + case SET_CURRENT_VIEW_VERSION: + writeSetCurrentViewVersionId( + (MetadataUpdate.SetCurrentViewVersion) metadataUpdate, generator); + break; default: throw new IllegalArgumentException( String.format( @@ -271,6 +289,10 @@ public static MetadataUpdate fromJson(JsonNode jsonNode) { return readRemoveProperties(jsonNode); case SET_LOCATION: return readSetLocation(jsonNode); + case ADD_VIEW_VERSION: + return readAddViewVersion(jsonNode); + case SET_CURRENT_VIEW_VERSION: + return readCurrentViewVersionId(jsonNode); default: throw new UnsupportedOperationException( String.format("Cannot convert metadata update action to json: %s", action)); @@ -384,6 +406,17 @@ private static void writeSetLocation(MetadataUpdate.SetLocation update, JsonGene gen.writeStringField(LOCATION, update.location()); } + private static void writeAddViewVersion( + MetadataUpdate.AddViewVersion metadataUpdate, JsonGenerator gen) throws IOException { + gen.writeFieldName(VIEW_VERSION); + ViewVersionParser.toJson(metadataUpdate.viewVersion(), gen); + } + + private static void writeSetCurrentViewVersionId( + MetadataUpdate.SetCurrentViewVersion metadataUpdate, JsonGenerator gen) throws IOException { + gen.writeNumberField(VIEW_VERSION_ID, metadataUpdate.versionId()); + } + private static MetadataUpdate readAssignUUID(JsonNode node) { String uuid = JsonUtil.getString(UUID, node); return new MetadataUpdate.AssignUUID(uuid); @@ -512,4 +545,13 @@ private static MetadataUpdate readSetLocation(JsonNode node) { String location = JsonUtil.getString(LOCATION, node); return new MetadataUpdate.SetLocation(location); } + + private static MetadataUpdate readAddViewVersion(JsonNode node) { + return new MetadataUpdate.AddViewVersion( + ViewVersionParser.fromJson(JsonUtil.get(VIEW_VERSION, node))); + } + + private static MetadataUpdate readCurrentViewVersionId(JsonNode node) { + return new MetadataUpdate.SetCurrentViewVersion(JsonUtil.getInt(VIEW_VERSION_ID, node)); + } } diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java index 00af4cf3a06c..d4df7169fd1e 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java @@ -19,27 +19,48 @@ package org.apache.iceberg.view; import java.io.Serializable; +import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.Schema; +import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.PropertyUtil; import org.immutables.value.Value; +import org.immutables.value.Value.Style.ImplementationVisibility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Value.Immutable +@SuppressWarnings("ImmutablesStyle") +@Value.Immutable(builder = false) +@Value.Style(allParameters = true, visibility = ImplementationVisibility.PACKAGE) public interface ViewMetadata extends Serializable { Logger LOG = LoggerFactory.getLogger(ViewMetadata.class); int SUPPORTED_VIEW_FORMAT_VERSION = 1; + int DEFAULT_VIEW_FORMAT_VERSION = 1; int formatVersion(); String location(); default Integer currentSchemaId() { - return currentVersion().schemaId(); + // fail when accessing the current schema if ViewMetadata was created through the + // ViewMetadataParser with an invalid schema id + int currentSchemaId = currentVersion().schemaId(); + Preconditions.checkArgument( + schemasById().containsKey(currentSchemaId), + "Cannot find current schema with id %s in schemas: %s", + currentSchemaId, + schemasById().keySet()); + + return currentSchemaId; } List schemas(); @@ -52,11 +73,21 @@ default Integer currentSchemaId() { Map properties(); + List changes(); + default ViewVersion version(int versionId) { return versionsById().get(versionId); } default ViewVersion currentVersion() { + // fail when accessing the current version if ViewMetadata was created through the + // ViewMetadataParser with an invalid view version id + Preconditions.checkArgument( + versionsById().containsKey(currentVersionId()), + "Cannot find current version %s in view versions: %s", + currentVersionId(), + versionsById().keySet()); + return versionsById().get(currentVersionId()); } @@ -85,47 +116,316 @@ default Schema schema() { } @Value.Check - default ViewMetadata checkAndNormalize() { + default void check() { Preconditions.checkArgument( formatVersion() > 0 && formatVersion() <= ViewMetadata.SUPPORTED_VIEW_FORMAT_VERSION, "Unsupported format version: %s", formatVersion()); + } - Preconditions.checkArgument(versions().size() > 0, "Invalid view versions: empty"); - Preconditions.checkArgument(history().size() > 0, "Invalid view history: empty"); - Preconditions.checkArgument(schemas().size() > 0, "Invalid schemas: empty"); + static Builder builder() { + return new Builder(); + } - Preconditions.checkArgument( - versionsById().containsKey(currentVersionId()), - "Cannot find current version %s in view versions: %s", - currentVersionId(), - versionsById().keySet()); + static Builder buildFrom(ViewMetadata base) { + return new Builder(base); + } - Preconditions.checkArgument( - schemasById().containsKey(currentSchemaId()), - "Cannot find current schema with id %s in schemas: %s", - currentSchemaId(), - schemasById().keySet()); + class Builder { + private static final int LAST_ADDED = -1; + private final List versions; + private final List schemas; + private final List history; + private final Map properties; + private final List changes; + private int formatVersion = DEFAULT_VIEW_FORMAT_VERSION; + private int currentVersionId; + private String location; + + // internal change tracking + private Integer lastAddedVersionId = null; + + // indexes + private final Map versionsById; + private final Map schemasById; + + private Builder() { + this.versions = Lists.newArrayList(); + this.versionsById = Maps.newHashMap(); + this.schemas = Lists.newArrayList(); + this.schemasById = Maps.newHashMap(); + this.history = Lists.newArrayList(); + this.properties = Maps.newHashMap(); + this.changes = Lists.newArrayList(); + } + + private Builder(ViewMetadata base) { + this.versions = Lists.newArrayList(base.versions()); + this.versionsById = Maps.newHashMap(base.versionsById()); + this.schemas = Lists.newArrayList(base.schemas()); + this.schemasById = Maps.newHashMap(base.schemasById()); + this.history = Lists.newArrayList(base.history()); + this.properties = Maps.newHashMap(base.properties()); + this.changes = Lists.newArrayList(); + this.formatVersion = base.formatVersion(); + this.currentVersionId = base.currentVersionId(); + this.location = base.location(); + } + + public Builder upgradeFormatVersion(int newFormatVersion) { + Preconditions.checkArgument( + newFormatVersion >= formatVersion, + "Cannot downgrade v%s view to v%s", + formatVersion, + newFormatVersion); + + if (formatVersion == newFormatVersion) { + return this; + } + + this.formatVersion = newFormatVersion; + changes.add(new MetadataUpdate.UpgradeFormatVersion(newFormatVersion)); + return this; + } + + public Builder setLocation(String newLocation) { + Preconditions.checkArgument(null != newLocation, "Invalid location: null"); + if (null != location && location.equals(newLocation)) { + return this; + } + + this.location = newLocation; + changes.add(new MetadataUpdate.SetLocation(newLocation)); + return this; + } + + public Builder setCurrentVersionId(int newVersionId) { + if (newVersionId == LAST_ADDED) { + ValidationException.check( + lastAddedVersionId != null, + "Cannot set last version id: no current version id has been set"); + return setCurrentVersionId(lastAddedVersionId); + } + + if (currentVersionId == newVersionId) { + return this; + } + + ViewVersion version = versionsById.get(newVersionId); + Preconditions.checkArgument( + version != null, "Cannot set current version to unknown version: %s", newVersionId); + + this.currentVersionId = newVersionId; + + if (lastAddedVersionId != null && lastAddedVersionId == newVersionId) { + changes.add(new MetadataUpdate.SetCurrentViewVersion(LAST_ADDED)); + } else { + changes.add(new MetadataUpdate.SetCurrentViewVersion(newVersionId)); + } + + return this; + } + + public Builder setCurrentVersion(ViewVersion version, Schema schema) { + int newSchemaId = addSchemaInternal(schema); + ViewVersion newVersion = + ImmutableViewVersion.builder().from(version).schemaId(newSchemaId).build(); + return setCurrentVersionId(addVersionInternal(newVersion)); + } + + public Builder addVersion(ViewVersion version) { + addVersionInternal(version); + return this; + } + + private int addVersionInternal(ViewVersion version) { + int newVersionId = reuseOrCreateNewViewVersionId(version); + if (versionsById.containsKey(newVersionId)) { + boolean addedInBuilder = + changes(MetadataUpdate.AddViewVersion.class) + .anyMatch(added -> added.viewVersion().versionId() == newVersionId); + this.lastAddedVersionId = addedInBuilder ? newVersionId : null; + return newVersionId; + } + + Preconditions.checkArgument( + schemasById.containsKey(version.schemaId()), + "Cannot add version with unknown schema: %s", + version.schemaId()); + + ViewVersion newVersion; + if (newVersionId != version.versionId()) { + newVersion = ImmutableViewVersion.builder().from(version).versionId(newVersionId).build(); + } else { + newVersion = version; + } + + versions.add(newVersion); + versionsById.put(newVersion.versionId(), newVersion); + changes.add(new MetadataUpdate.AddViewVersion(newVersion)); + history.add( + ImmutableViewHistoryEntry.builder() + .timestampMillis(newVersion.timestampMillis()) + .versionId(newVersion.versionId()) + .build()); + + this.lastAddedVersionId = newVersionId; + + return newVersionId; + } + + private int reuseOrCreateNewViewVersionId(ViewVersion viewVersion) { + // if the view version already exists, use its id; otherwise use the highest id + 1 + int newVersionId = viewVersion.versionId(); + for (ViewVersion version : versions) { + if (version.equals(viewVersion)) { + return version.versionId(); + } else if (version.versionId() >= newVersionId) { + newVersionId = viewVersion.versionId() + 1; + } + } + + return newVersionId; + } + + public Builder addSchema(Schema schema) { + addSchemaInternal(schema); + return this; + } + + private int addSchemaInternal(Schema schema) { + int newSchemaId = reuseOrCreateNewSchemaId(schema); + if (schemasById.containsKey(newSchemaId)) { + // this schema existed or was already added in the builder + return newSchemaId; + } + + Schema newSchema; + if (newSchemaId != schema.schemaId()) { + newSchema = new Schema(newSchemaId, schema.columns(), schema.identifierFieldIds()); + } else { + newSchema = schema; + } + + int highestFieldId = Math.max(highestFieldId(), newSchema.highestFieldId()); + schemas.add(newSchema); + schemasById.put(newSchema.schemaId(), newSchema); + changes.add(new MetadataUpdate.AddSchema(newSchema, highestFieldId)); + + return newSchemaId; + } + + private int highestFieldId() { + return schemas.stream().map(Schema::highestFieldId).max(Integer::compareTo).orElse(0); + } - int versionHistorySizeToKeep = - PropertyUtil.propertyAsInt( - properties(), - ViewProperties.VERSION_HISTORY_SIZE, - ViewProperties.VERSION_HISTORY_SIZE_DEFAULT); + private int reuseOrCreateNewSchemaId(Schema newSchema) { + // if the schema already exists, use its id; otherwise use the highest id + 1 + int newSchemaId = newSchema.schemaId(); + for (Schema schema : schemas) { + if (schema.sameSchema(newSchema)) { + return schema.schemaId(); + } else if (schema.schemaId() >= newSchemaId) { + newSchemaId = schema.schemaId() + 1; + } + } - if (versionHistorySizeToKeep <= 0) { - LOG.warn( - "{} must be positive but was {}", + return newSchemaId; + } + + public Builder setProperties(Map updated) { + if (updated.isEmpty()) { + return this; + } + + properties.putAll(updated); + changes.add(new MetadataUpdate.SetProperties(updated)); + return this; + } + + public Builder removeProperties(Set propertiesToRemove) { + if (propertiesToRemove.isEmpty()) { + return this; + } + + propertiesToRemove.forEach(properties::remove); + changes.add(new MetadataUpdate.RemoveProperties(propertiesToRemove)); + return this; + } + + public ViewMetadata build() { + Preconditions.checkArgument(null != location, "Invalid location: null"); + Preconditions.checkArgument(versions.size() > 0, "Invalid view: no versions were added"); + + int historySize = + PropertyUtil.propertyAsInt( + properties, + ViewProperties.VERSION_HISTORY_SIZE, + ViewProperties.VERSION_HISTORY_SIZE_DEFAULT); + + Preconditions.checkArgument( + historySize > 0, + "%s must be positive but was %s", ViewProperties.VERSION_HISTORY_SIZE, - versionHistorySizeToKeep); - } else if (versions().size() > versionHistorySizeToKeep) { - List versions = - versions().subList(versions().size() - versionHistorySizeToKeep, versions().size()); - List history = - history().subList(history().size() - versionHistorySizeToKeep, history().size()); - return ImmutableViewMetadata.builder().from(this).versions(versions).history(history).build(); + historySize); + + // expire old versions, but keep at least the versions added in this builder + int numAddedVersions = (int) changes(MetadataUpdate.AddViewVersion.class).count(); + int numVersionsToKeep = Math.max(numAddedVersions, historySize); + + List retainedVersions; + List retainedHistory; + if (versions.size() > numVersionsToKeep) { + retainedVersions = expireVersions(versionsById, numVersionsToKeep); + Set retainedVersionIds = + retainedVersions.stream().map(ViewVersion::versionId).collect(Collectors.toSet()); + retainedHistory = updateHistory(history, retainedVersionIds); + } else { + retainedVersions = versions; + retainedHistory = history; + } + + return ImmutableViewMetadata.of( + formatVersion, + location, + schemas, + currentVersionId, + retainedVersions, + retainedHistory, + properties, + changes); + } + + static List expireVersions( + Map versionsById, int numVersionsToKeep) { + // version ids are assigned sequentially. keep the latest versions by ID. + List ids = Lists.newArrayList(versionsById.keySet()); + ids.sort(Comparator.reverseOrder()); + + List retainedVersions = Lists.newArrayList(); + for (int idToKeep : ids.subList(0, numVersionsToKeep)) { + retainedVersions.add(versionsById.get(idToKeep)); + } + + return retainedVersions; } - return this; + static List updateHistory(List history, Set ids) { + List retainedHistory = Lists.newArrayList(); + for (ViewHistoryEntry entry : history) { + if (ids.contains(entry.versionId())) { + retainedHistory.add(entry); + } else { + // clear history past any unknown version + retainedHistory.clear(); + } + } + + return retainedHistory; + } + + private Stream changes(Class updateClass) { + return changes.stream().filter(updateClass::isInstance).map(updateClass::cast); + } } } diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java index f1fca511b1dc..c994c82ea875 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java @@ -33,6 +33,7 @@ import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.util.JsonUtil; @@ -129,15 +130,15 @@ public static ViewMetadata fromJson(JsonNode json) { historyEntries.add(ViewHistoryEntryParser.fromJson(vLog)); } - return ImmutableViewMetadata.builder() - .location(location) - .currentVersionId(currentVersionId) - .properties(properties) - .versions(versions) - .schemas(schemas) - .history(historyEntries) - .formatVersion(formatVersion) - .build(); + return ImmutableViewMetadata.of( + formatVersion, + location, + schemas, + currentVersionId, + versions, + historyEntries, + properties, + ImmutableList.of()); } public static void overwrite(ViewMetadata metadata, OutputFile outputFile) { diff --git a/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java b/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java index 9d2889a6a5bf..8bdbdc431c98 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java @@ -28,7 +28,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.util.JsonUtil; -class ViewVersionParser { +public class ViewVersionParser { private static final String VERSION_ID = "version-id"; private static final String TIMESTAMP_MS = "timestamp-ms"; @@ -40,7 +40,7 @@ class ViewVersionParser { private ViewVersionParser() {} - static void toJson(ViewVersion version, JsonGenerator generator) throws IOException { + public static void toJson(ViewVersion version, JsonGenerator generator) throws IOException { Preconditions.checkArgument(version != null, "Cannot serialize null view version"); generator.writeStartObject(); @@ -74,7 +74,7 @@ static ViewVersion fromJson(String json) { return JsonUtil.parse(json, ViewVersionParser::fromJson); } - static ViewVersion fromJson(JsonNode node) { + public static ViewVersion fromJson(JsonNode node) { Preconditions.checkArgument(node != null, "Cannot parse view version from null object"); Preconditions.checkArgument( node.isObject(), "Cannot parse view version from a non-object: %s", node); diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java index 09259aaa3752..fe36c5daa26d 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.IntStream; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; @@ -35,6 +36,8 @@ import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.Pair; +import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.iceberg.view.ViewVersion; import org.assertj.core.api.Assertions; import org.junit.Assert; import org.junit.Rule; @@ -822,6 +825,65 @@ public void testRemoveStatistics() { MetadataUpdateParser.toJson(expected)); } + /** AddViewVersion */ + @Test + public void testAddViewVersionFromJson() { + String action = MetadataUpdateParser.ADD_VIEW_VERSION; + long timestamp = 123456789; + ViewVersion viewVersion = + ImmutableViewVersion.builder() + .versionId(23) + .timestampMillis(timestamp) + .schemaId(4) + .putSummary("operation", "replace") + .defaultNamespace(Namespace.of("ns")) + .build(); + String json = + String.format( + "{\"action\":\"%s\",\"view-version\":{\"version-id\":23,\"timestamp-ms\":123456789,\"schema-id\":4,\"summary\":{\"operation\":\"replace\"},\"default-namespace\":[\"ns\"],\"representations\":[]}}", + action); + MetadataUpdate expected = new MetadataUpdate.AddViewVersion(viewVersion); + assertEquals(action, expected, MetadataUpdateParser.fromJson(json)); + } + + @Test + public void testAddViewVersionToJson() { + String action = MetadataUpdateParser.ADD_VIEW_VERSION; + long timestamp = 123456789; + ViewVersion viewVersion = + ImmutableViewVersion.builder() + .versionId(23) + .timestampMillis(timestamp) + .schemaId(4) + .putSummary("operation", "replace") + .defaultNamespace(Namespace.of("ns")) + .build(); + String expected = + String.format( + "{\"action\":\"%s\",\"view-version\":{\"version-id\":23,\"timestamp-ms\":123456789,\"schema-id\":4,\"summary\":{\"operation\":\"replace\"},\"default-namespace\":[\"ns\"],\"representations\":[]}}", + action); + + MetadataUpdate update = new MetadataUpdate.AddViewVersion(viewVersion); + Assertions.assertThat(MetadataUpdateParser.toJson(update)).isEqualTo(expected); + } + + /** SetCurrentViewVersion */ + @Test + public void testSetCurrentViewVersionFromJson() { + String action = MetadataUpdateParser.SET_CURRENT_VIEW_VERSION; + String json = String.format("{\"action\":\"%s\",\"view-version-id\":23}", action); + MetadataUpdate expected = new MetadataUpdate.SetCurrentViewVersion(23); + assertEquals(action, expected, MetadataUpdateParser.fromJson(json)); + } + + @Test + public void testSetCurrentViewVersionToJson() { + String action = MetadataUpdateParser.SET_CURRENT_VIEW_VERSION; + String expected = String.format("{\"action\":\"%s\",\"view-version-id\":23}", action); + MetadataUpdate update = new MetadataUpdate.SetCurrentViewVersion(23); + Assertions.assertThat(MetadataUpdateParser.toJson(update)).isEqualTo(expected); + } + public void assertEquals( String action, MetadataUpdate expectedUpdate, MetadataUpdate actualUpdate) { switch (action) { @@ -906,6 +968,16 @@ public void assertEquals( assertEqualsSetLocation( (MetadataUpdate.SetLocation) expectedUpdate, (MetadataUpdate.SetLocation) actualUpdate); break; + case MetadataUpdateParser.ADD_VIEW_VERSION: + assertEqualsAddViewVersion( + (MetadataUpdate.AddViewVersion) expectedUpdate, + (MetadataUpdate.AddViewVersion) actualUpdate); + break; + case MetadataUpdateParser.SET_CURRENT_VIEW_VERSION: + assertEqualsSetCurrentViewVersion( + (MetadataUpdate.SetCurrentViewVersion) expectedUpdate, + (MetadataUpdate.SetCurrentViewVersion) actualUpdate); + break; default: Assert.fail("Unrecognized metadata update action: " + action); } @@ -1147,6 +1219,16 @@ private static void assertEqualsSetLocation( Assert.assertEquals("Location should be the same", expected.location(), actual.location()); } + private static void assertEqualsAddViewVersion( + MetadataUpdate.AddViewVersion expected, MetadataUpdate.AddViewVersion actual) { + Assertions.assertThat(actual.viewVersion()).isEqualTo(expected.viewVersion()); + } + + private static void assertEqualsSetCurrentViewVersion( + MetadataUpdate.SetCurrentViewVersion expected, MetadataUpdate.SetCurrentViewVersion actual) { + Assertions.assertThat(actual.versionId()).isEqualTo(expected.versionId()); + } + private String createManifestListWithManifestFiles(long snapshotId, Long parentSnapshotId) throws IOException { File manifestList = temp.newFile("manifests" + UUID.randomUUID()); diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java b/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java index 5d4b3bd0f30a..a852a716d53f 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java +++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java @@ -21,78 +21,136 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.types.Types; +import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.Test; public class TestViewMetadata { + private ViewVersion newViewVersion(int id, String sql) { + return ImmutableViewVersion.builder() + .versionId(id) + .timestampMillis(System.currentTimeMillis()) + .defaultCatalog("prod") + .defaultNamespace(Namespace.of("default")) + .summary(ImmutableMap.of("operation", "create")) + .schemaId(1) + .build(); + } + @Test - public void nullAndMissingFields() { - assertThatThrownBy(() -> ImmutableViewMetadata.builder().build()) - .isInstanceOf(IllegalStateException.class) - .hasMessage( - "Cannot build ViewMetadata, some of required attributes are not set [formatVersion, location, currentVersionId]"); + public void testExpiration() { + // purposely use versions and timestamps that do not match to check that version ID is used + ViewVersion v1 = newViewVersion(1, "select 1 as count"); + ViewVersion v3 = newViewVersion(3, "select count from t1"); + ViewVersion v2 = newViewVersion(2, "select count(1) as count from t2"); + Map versionsById = ImmutableMap.of(1, v1, 2, v2, 3, v3); - assertThatThrownBy(() -> ImmutableViewMetadata.builder().formatVersion(1).build()) - .isInstanceOf(IllegalStateException.class) - .hasMessage( - "Cannot build ViewMetadata, some of required attributes are not set [location, currentVersionId]"); + List retainedVersions = ViewMetadata.Builder.expireVersions(versionsById, 2); + assertThat(retainedVersions).hasSameElementsAs(ImmutableList.of(v2, v3)); + } - assertThatThrownBy( - () -> ImmutableViewMetadata.builder().formatVersion(1).location("location").build()) - .isInstanceOf(IllegalStateException.class) - .hasMessage( - "Cannot build ViewMetadata, some of required attributes are not set [currentVersionId]"); + @Test + public void testUpdateHistory() { + ViewVersion v1 = newViewVersion(1, "select 1 as count"); + ViewVersion v2 = newViewVersion(2, "select count(1) as count from t2"); + ViewVersion v3 = newViewVersion(3, "select count from t1"); + + Set versionsById = ImmutableSet.of(2, 3); + + List history = + ImmutableList.of( + ImmutableViewHistoryEntry.builder() + .versionId(v1.versionId()) + .timestampMillis(v1.timestampMillis()) + .build(), + ImmutableViewHistoryEntry.builder() + .versionId(v2.versionId()) + .timestampMillis(v2.timestampMillis()) + .build(), + ImmutableViewHistoryEntry.builder() + .versionId(v3.versionId()) + .timestampMillis(v3.timestampMillis()) + .build()); + + List retainedHistory = + ViewMetadata.Builder.updateHistory(history, versionsById); + assertThat(retainedHistory).hasSameElementsAs(history.subList(1, 3)); + } + + @Test + public void nullAndMissingFields() { + assertThatThrownBy(() -> ViewMetadata.builder().build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid location: null"); + + assertThatThrownBy(() -> ViewMetadata.builder().setLocation("location").build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid view: no versions were added"); assertThatThrownBy( - () -> - ImmutableViewMetadata.builder() - .formatVersion(1) - .location("location") - .currentVersionId(1) - .build()) + () -> ViewMetadata.builder().setLocation("location").setCurrentVersionId(1).build()) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid view versions: empty"); + .hasMessage("Cannot set current version to unknown version: 1"); } @Test public void unsupportedFormatVersion() { assertThatThrownBy( () -> - ImmutableViewMetadata.builder() - .formatVersion(23) - .location("location") - .currentVersionId(1) + ViewMetadata.builder() + .upgradeFormatVersion(23) + .setLocation("location") + .addSchema( + new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(1) + .timestampMillis(23L) + .putSummary("operation", "op") + .defaultNamespace(Namespace.of("ns")) + .build()) + .setCurrentVersionId(1) .build()) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Unsupported format version: 23"); + + assertThatThrownBy( + () -> + ViewMetadata.builder() + .upgradeFormatVersion(0) + .setLocation("location") + .setCurrentVersionId(1) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot downgrade v1 view to v0"); } @Test public void emptyViewVersion() { assertThatThrownBy( - () -> - ImmutableViewMetadata.builder() - .formatVersion(1) - .location("location") - .currentVersionId(1) - .build()) + () -> ViewMetadata.builder().setLocation("location").setCurrentVersionId(1).build()) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid view versions: empty"); + .hasMessage("Cannot set current version to unknown version: 1"); } @Test public void emptySchemas() { assertThatThrownBy( () -> - ImmutableViewMetadata.builder() - .formatVersion(1) - .location("location") - .currentVersionId(1) - .addVersions( + ViewMetadata.builder() + .setLocation("location") + .addVersion( ImmutableViewVersion.builder() .schemaId(1) .versionId(1) @@ -100,25 +158,21 @@ public void emptySchemas() { .putSummary("operation", "op") .defaultNamespace(Namespace.of("ns")) .build()) - .addHistory( - ImmutableViewHistoryEntry.builder() - .timestampMillis(23L) - .versionId(1) - .build()) + .setCurrentVersionId(1) .build()) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid schemas: empty"); + .hasMessage("Cannot add version with unknown schema: 1"); } @Test public void invalidCurrentVersionId() { assertThatThrownBy( () -> - ImmutableViewMetadata.builder() - .formatVersion(1) - .location("location") - .currentVersionId(23) - .addVersions( + ViewMetadata.builder() + .setLocation("location") + .addSchema( + new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( ImmutableViewVersion.builder() .schemaId(1) .versionId(1) @@ -126,27 +180,21 @@ public void invalidCurrentVersionId() { .putSummary("operation", "op") .defaultNamespace(Namespace.of("ns")) .build()) - .addHistory( - ImmutableViewHistoryEntry.builder() - .timestampMillis(23L) - .versionId(1) - .build()) - .addSchemas( - new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .setCurrentVersionId(23) .build()) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Cannot find current version 23 in view versions: [1]"); + .hasMessage("Cannot set current version to unknown version: 23"); } @Test public void invalidCurrentSchemaId() { assertThatThrownBy( () -> - ImmutableViewMetadata.builder() - .formatVersion(1) - .location("location") - .currentVersionId(1) - .addVersions( + ViewMetadata.builder() + .setLocation("location") + .addSchema( + new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( ImmutableViewVersion.builder() .schemaId(23) .versionId(1) @@ -154,133 +202,267 @@ public void invalidCurrentSchemaId() { .timestampMillis(23L) .putSummary("operation", "op") .build()) - .addHistory( - ImmutableViewHistoryEntry.builder() - .timestampMillis(23L) - .versionId(1) - .build()) - .addSchemas( - new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .setCurrentVersionId(1) .build()) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Cannot find current schema with id 23 in schemas: [1]"); + .hasMessage("Cannot add version with unknown schema: 23"); } @Test public void invalidVersionHistorySizeToKeep() { - ImmutableViewMetadata viewMetadata = - ImmutableViewMetadata.builder() - // setting history to < 1 shouldn't do anything and only issue a WARN - .properties(ImmutableMap.of(ViewProperties.VERSION_HISTORY_SIZE, "0")) - .formatVersion(1) - .location("location") - .currentVersionId(3) - .addVersions( - ImmutableViewVersion.builder() - .schemaId(1) - .versionId(1) - .timestampMillis(23L) - .putSummary("operation", "a") - .defaultNamespace(Namespace.of("ns")) - .build()) - .addVersions( - ImmutableViewVersion.builder() - .schemaId(1) - .versionId(2) - .timestampMillis(24L) - .putSummary("operation", "b") - .defaultNamespace(Namespace.of("ns")) - .build()) - .addVersions( - ImmutableViewVersion.builder() - .schemaId(1) - .versionId(3) - .timestampMillis(25L) - .putSummary("operation", "c") - .defaultNamespace(Namespace.of("ns")) - .build()) - .addHistory( - ImmutableViewHistoryEntry.builder().versionId(1).timestampMillis(23L).build()) - .addHistory( - ImmutableViewHistoryEntry.builder().versionId(2).timestampMillis(24L).build()) - .addSchemas(new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) - .build(); - - assertThat(viewMetadata.versions()).hasSize(3); - assertThat(viewMetadata.history()).hasSize(2); - } - - @Test - public void emptyHistory() { assertThatThrownBy( () -> - ImmutableViewMetadata.builder() - .formatVersion(1) - .location("location") - .currentVersionId(2) - .addVersions( + ViewMetadata.builder() + .setProperties(ImmutableMap.of(ViewProperties.VERSION_HISTORY_SIZE, "0")) + .setLocation("location") + .addSchema( + new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( ImmutableViewVersion.builder() .schemaId(1) .versionId(1) .timestampMillis(23L) - .putSummary("operation", "op") + .putSummary("operation", "a") .defaultNamespace(Namespace.of("ns")) .build()) - .addVersions( + .addVersion( ImmutableViewVersion.builder() .schemaId(1) .versionId(2) .timestampMillis(24L) - .putSummary("operation", "op") + .putSummary("operation", "b") .defaultNamespace(Namespace.of("ns")) .build()) - .addSchemas( - new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(3) + .timestampMillis(25L) + .putSummary("operation", "c") + .defaultNamespace(Namespace.of("ns")) + .build()) + .setCurrentVersionId(3) .build()) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid view history: empty"); + .hasMessage("version.history.num-entries must be positive but was 0"); } @Test public void viewHistoryNormalization() { - ImmutableViewMetadata viewMetadata = - ImmutableViewMetadata.builder() - .properties(ImmutableMap.of(ViewProperties.VERSION_HISTORY_SIZE, "1")) - .formatVersion(1) - .location("location") - .currentVersionId(3) - .addVersions( - ImmutableViewVersion.builder() - .schemaId(1) - .versionId(1) - .timestampMillis(23L) - .putSummary("operation", "a") - .defaultNamespace(Namespace.of("ns")) - .build()) - .addVersions( - ImmutableViewVersion.builder() - .schemaId(1) - .versionId(2) - .timestampMillis(24L) - .putSummary("operation", "b") - .defaultNamespace(Namespace.of("ns")) - .build()) - .addVersions( - ImmutableViewVersion.builder() - .schemaId(1) - .versionId(3) - .timestampMillis(25L) - .putSummary("operation", "c") - .defaultNamespace(Namespace.of("ns")) - .build()) - .addHistory( - ImmutableViewHistoryEntry.builder().versionId(1).timestampMillis(23L).build()) - .addHistory( - ImmutableViewHistoryEntry.builder().versionId(2).timestampMillis(24L).build()) - .addSchemas(new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + Map properties = ImmutableMap.of(ViewProperties.VERSION_HISTORY_SIZE, "2"); + ViewVersion viewVersionOne = + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(1) + .timestampMillis(23L) + .putSummary("operation", "a") + .defaultNamespace(Namespace.of("ns")) + .build(); + ViewVersion viewVersionTwo = + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(2) + .timestampMillis(24L) + .putSummary("operation", "b") + .defaultNamespace(Namespace.of("ns")) + .build(); + ViewVersion viewVersionThree = + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(3) + .timestampMillis(25L) + .putSummary("operation", "c") + .defaultNamespace(Namespace.of("ns")) + .build(); + + ViewMetadata originalViewMetadata = + ViewMetadata.builder() + .setProperties(properties) + .setLocation("location") + .addSchema(new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion(viewVersionOne) + .addVersion(viewVersionTwo) + .addVersion(viewVersionThree) + .setCurrentVersionId(3) + .build(); + + // the first build will not expire versions that were added in the builder + assertThat(originalViewMetadata.versions()).hasSize(3); + assertThat(originalViewMetadata.history()).hasSize(3); + + // rebuild the metadata to expire older versions + ViewMetadata viewMetadata = ViewMetadata.buildFrom(originalViewMetadata).build(); + assertThat(viewMetadata.versions()).hasSize(2); + assertThat(viewMetadata.history()).hasSize(2); + + // make sure that metadata changes reflect the current state after the history was adjusted, + // meaning that the first view version shouldn't be included + List changes = originalViewMetadata.changes(); + assertThat(changes).hasSize(7); + assertThat(changes) + .element(0) + .isInstanceOf(MetadataUpdate.SetProperties.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetProperties.class)) + .extracting(MetadataUpdate.SetProperties::updated) + .isEqualTo(properties); + + assertThat(changes) + .element(1) + .isInstanceOf(MetadataUpdate.SetLocation.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetLocation.class)) + .extracting(MetadataUpdate.SetLocation::location) + .isEqualTo("location"); + + assertThat(changes) + .element(2) + .isInstanceOf(MetadataUpdate.AddSchema.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddSchema.class)) + .extracting(MetadataUpdate.AddSchema::schema) + .extracting(Schema::schemaId) + .isEqualTo(1); + + assertThat(changes) + .element(3) + .isInstanceOf(MetadataUpdate.AddViewVersion.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) + .extracting(MetadataUpdate.AddViewVersion::viewVersion) + .isEqualTo(viewVersionOne); + + assertThat(changes) + .element(4) + .isInstanceOf(MetadataUpdate.AddViewVersion.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) + .extracting(MetadataUpdate.AddViewVersion::viewVersion) + .isEqualTo(viewVersionTwo); + + assertThat(changes) + .element(5) + .isInstanceOf(MetadataUpdate.AddViewVersion.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) + .extracting(MetadataUpdate.AddViewVersion::viewVersion) + .isEqualTo(viewVersionThree); + + assertThat(changes) + .element(6) + .isInstanceOf(MetadataUpdate.SetCurrentViewVersion.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetCurrentViewVersion.class)) + .extracting(MetadataUpdate.SetCurrentViewVersion::versionId) + .isEqualTo(-1); + } + + @Test + public void viewMetadataAndMetadataChanges() { + Map properties = ImmutableMap.of("key1", "prop1", "key2", "prop2"); + Schema schemaOne = new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get())); + Schema schemaTwo = new Schema(2, Types.NestedField.required(1, "y", Types.LongType.get())); + ViewVersion viewVersionOne = + ImmutableViewVersion.builder() + .schemaId(schemaOne.schemaId()) + .versionId(1) + .timestampMillis(23L) + .putSummary("operation", "a") + .defaultNamespace(Namespace.of("ns")) .build(); + ViewVersion viewVersionTwo = + ImmutableViewVersion.builder() + .schemaId(schemaOne.schemaId()) + .versionId(2) + .timestampMillis(24L) + .putSummary("operation", "b") + .defaultNamespace(Namespace.of("ns")) + .build(); + ViewVersion viewVersionThree = + ImmutableViewVersion.builder() + .schemaId(schemaTwo.schemaId()) + .versionId(3) + .timestampMillis(25L) + .putSummary("operation", "c") + .defaultNamespace(Namespace.of("ns")) + .build(); + + ViewMetadata viewMetadata = + ViewMetadata.builder() + .setLocation("custom-location") + .setProperties(properties) + .addSchema(schemaOne) + .addSchema(schemaTwo) + .addVersion(viewVersionOne) + .addVersion(viewVersionTwo) + .addVersion(viewVersionThree) + .setCurrentVersionId(3) + .build(); + + assertThat(viewMetadata.versions()) + .hasSize(3) + .containsExactly(viewVersionOne, viewVersionTwo, viewVersionThree); + assertThat(viewMetadata.history()).hasSize(3); + assertThat(viewMetadata.currentVersionId()).isEqualTo(3); + assertThat(viewMetadata.currentVersion()).isEqualTo(viewVersionThree); + assertThat(viewMetadata.formatVersion()).isEqualTo(ViewMetadata.DEFAULT_VIEW_FORMAT_VERSION); + assertThat(viewMetadata.schemas()).hasSize(2).containsExactly(schemaOne, schemaTwo); + assertThat(viewMetadata.schema().asStruct()).isEqualTo(schemaTwo.asStruct()); + assertThat(viewMetadata.currentSchemaId()).isEqualTo(schemaTwo.schemaId()); + assertThat(viewMetadata.location()).isEqualTo("custom-location"); + assertThat(viewMetadata.properties()).isEqualTo(properties); + + List changes = viewMetadata.changes(); + assertThat(changes).hasSize(8); + assertThat(changes) + .element(0) + .isInstanceOf(MetadataUpdate.SetLocation.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetLocation.class)) + .extracting(MetadataUpdate.SetLocation::location) + .isEqualTo("custom-location"); + + assertThat(changes) + .element(1) + .isInstanceOf(MetadataUpdate.SetProperties.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetProperties.class)) + .extracting(MetadataUpdate.SetProperties::updated) + .isEqualTo(properties); + + assertThat(changes) + .element(2) + .isInstanceOf(MetadataUpdate.AddSchema.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddSchema.class)) + .extracting(MetadataUpdate.AddSchema::schema) + .extracting(Schema::schemaId) + .isEqualTo(1); + + assertThat(changes) + .element(3) + .isInstanceOf(MetadataUpdate.AddSchema.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddSchema.class)) + .extracting(MetadataUpdate.AddSchema::schema) + .extracting(Schema::schemaId) + .isEqualTo(2); + + assertThat(changes) + .element(4) + .isInstanceOf(MetadataUpdate.AddViewVersion.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) + .extracting(MetadataUpdate.AddViewVersion::viewVersion) + .isEqualTo(viewVersionOne); + + assertThat(changes) + .element(5) + .isInstanceOf(MetadataUpdate.AddViewVersion.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) + .extracting(MetadataUpdate.AddViewVersion::viewVersion) + .isEqualTo(viewVersionTwo); + + assertThat(changes) + .element(6) + .isInstanceOf(MetadataUpdate.AddViewVersion.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) + .extracting(MetadataUpdate.AddViewVersion::viewVersion) + .isEqualTo(viewVersionThree); - assertThat(viewMetadata.versions()).hasSize(1); - assertThat(viewMetadata.history()).hasSize(1); + assertThat(changes) + .element(7) + .isInstanceOf(MetadataUpdate.SetCurrentViewVersion.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetCurrentViewVersion.class)) + .extracting(MetadataUpdate.SetCurrentViewVersion::versionId) + .isEqualTo(-1); } } diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java index 6a046d980f03..807a1df9b1d1 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java @@ -26,7 +26,6 @@ import java.nio.file.Paths; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; @@ -72,9 +71,6 @@ public void readAndWriteValidViewMetadata() throws Exception { .build()) .build(); - ViewHistoryEntry historyEntry1 = - ImmutableViewHistoryEntry.builder().timestampMillis(4353L).versionId(1).build(); - ViewVersion version2 = ImmutableViewVersion.builder() .versionId(2) @@ -90,25 +86,23 @@ public void readAndWriteValidViewMetadata() throws Exception { .build()) .build(); - ViewHistoryEntry historyEntry2 = - ImmutableViewHistoryEntry.builder().timestampMillis(5555L).versionId(2).build(); - String json = readViewMetadataInputFile("org/apache/iceberg/view/ValidViewMetadata.json"); ViewMetadata expectedViewMetadata = - ImmutableViewMetadata.builder() - .schemas(ImmutableList.of(TEST_SCHEMA)) - .versions(ImmutableList.of(version1, version2)) - .history(ImmutableList.of(historyEntry1, historyEntry2)) - .location("s3://bucket/test/location") - .properties(ImmutableMap.of("some-key", "some-value")) - .currentVersionId(2) - .formatVersion(1) + ViewMetadata.builder() + .addSchema(TEST_SCHEMA) + .addVersion(version1) + .addVersion(version2) + .setLocation("s3://bucket/test/location") + .setProperties(ImmutableMap.of("some-key", "some-value")) + .setCurrentVersionId(2) + .upgradeFormatVersion(1) .build(); ViewMetadata actual = ViewMetadataParser.fromJson(json); assertThat(actual) .usingRecursiveComparison() .ignoringFieldsOfTypes(Schema.class) + .ignoringFields("changes") .isEqualTo(expectedViewMetadata); for (Schema schema : expectedViewMetadata.schemas()) { assertThat(schema.sameSchema(actual.schemasById().get(schema.schemaId()))).isTrue(); @@ -118,22 +112,13 @@ public void readAndWriteValidViewMetadata() throws Exception { assertThat(actual) .usingRecursiveComparison() .ignoringFieldsOfTypes(Schema.class) + .ignoringFields("changes") .isEqualTo(expectedViewMetadata); for (Schema schema : expectedViewMetadata.schemas()) { assertThat(schema.sameSchema(actual.schemasById().get(schema.schemaId()))).isTrue(); } } - @Test - public void readViewMetadataWithLimitedNumberVersionEntries() throws Exception { - String json = - readViewMetadataInputFile("org/apache/iceberg/view/ViewMetadataLimitedVersions.json"); - - ViewMetadata viewMetadata = ViewMetadataParser.fromJson(json); - assertThat(viewMetadata.versions()).hasSize(1); - assertThat(viewMetadata.history()).hasSize(1); - } - @Test public void failReadingViewMetadataMissingLocation() throws Exception { String json = @@ -147,7 +132,8 @@ public void failReadingViewMetadataMissingLocation() throws Exception { public void failReadingViewMetadataInvalidSchemaId() throws Exception { String json = readViewMetadataInputFile("org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json"); - assertThatThrownBy(() -> ViewMetadataParser.fromJson(json)) + ViewMetadata metadata = ViewMetadataParser.fromJson(json); + assertThatThrownBy(metadata::currentSchemaId) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot find current schema with id 1234 in schemas: [1]"); } @@ -165,7 +151,8 @@ public void failReadingViewMetadataMissingVersion() throws Exception { public void failReadingViewMetadataInvalidVersionId() throws Exception { String json = readViewMetadataInputFile("org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json"); - assertThatThrownBy(() -> ViewMetadataParser.fromJson(json)) + ViewMetadata metadata = ViewMetadataParser.fromJson(json); + assertThatThrownBy(metadata::currentVersion) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot find current version 1234 in view versions: [1, 2]"); } diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataLimitedVersions.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataLimitedVersions.json deleted file mode 100644 index db80a3baefb7..000000000000 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataLimitedVersions.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "format-version": 1, - "location": "s3://bucket/test/location", - "properties": {"version.history.num-entries": "1"}, - "current-schema-id": 1, - "schemas": [ - { - "type": "struct", - "schema-id": 1, - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - }, - { - "id": 2, - "name": "y", - "required": true, - "type": "long", - "doc": "comment" - }, - { - "id": 3, - "name": "z", - "required": true, - "type": "long" - } - ] - } - ], - "current-version-id": 2, - "versions": [ - { - "version-id": 1, - "timestamp-ms": 4353, - "summary": {"operation":"create"}, - "schema-id": 1, - "default-catalog": "some-catalog", - "default-namespace": [], - "representations": [ - { - "type": "sql", - "sql": "select 'foo' foo", - "dialect": "spark-sql" - } - ] - }, - { - "version-id": 2, - "timestamp-ms": 5555, - "summary": {"operation": "replace"}, - "schema-id": 1, - "default-catalog": "some-catalog", - "default-namespace": [], - "representations": [ - { - "type": "sql", - "sql": "select 1 id, 'abc' data", - "dialect": "spark-sql" - } - ] - } - ], - "version-log": [ - { - "timestamp-ms": 4353, - "version-id": 1 - }, - { - "timestamp-ms": 5555, - "version-id": 2 - } - ] -} \ No newline at end of file