Skip to content

Core: Add metadata updates for Views / Use own Builder for ViewMetadata #8147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rdblue merged 7 commits into from
Sep 4, 2023
Merged

Core: Add metadata updates for Views / Use own Builder for ViewMetadata #8147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a2c2309
Core: Add metadata updates for Views / use own Builder for ViewMetadata
nastra Jul 25, 2023
e6c0b3e
Move validations into Builder to be more lenient when ViewMetadata is…
nastra Aug 16, 2023
6ab376b
review feedback
nastra Aug 17, 2023
7eda8ea
move validation when setting current version id
nastra Aug 17, 2023
fb378f5
more changes
nastra Aug 17, 2023
f425b1d
Core: Update ViewMetadata builder and tests.
rdblue Sep 3, 2023
a5a8ad0
fixes
nastra Sep 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 71 additions & 2 deletions core/src/main/java/org/apache/iceberg/MetadataUpdate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,20 @@
import java.util.Map;
import java.util.Set;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.view.ViewMetadata;
import org.apache.iceberg.view.ViewVersion;

/** Represents a change to table metadata. */
/** Represents a change to table or view metadata. */
public interface MetadataUpdate extends Serializable {
void applyTo(TableMetadata.Builder metadataBuilder);
default void applyTo(TableMetadata.Builder metadataBuilder) {
throw new UnsupportedOperationException(
String.format("Cannot apply update %s to a table", this.getClass().getSimpleName()));
}

default void applyTo(ViewMetadata.Builder viewMetadataBuilder) {
throw new UnsupportedOperationException(
String.format("Cannot apply update %s to a view", this.getClass().getSimpleName()));
}

class AssignUUID implements MetadataUpdate {
private final String uuid;
Expand Down Expand Up @@ -60,6 +70,11 @@ public int formatVersion() {
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.upgradeFormatVersion(formatVersion);
}

@Override
public void applyTo(ViewMetadata.Builder viewMetadataBuilder) {
viewMetadataBuilder.upgradeFormatVersion(formatVersion);
}
}

class AddSchema implements MetadataUpdate {
Expand All @@ -83,6 +98,11 @@ public int lastColumnId() {
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addSchema(schema, lastColumnId);
}

@Override
public void applyTo(ViewMetadata.Builder viewMetadataBuilder) {
viewMetadataBuilder.addSchema(schema);
}
}

class SetCurrentSchema implements MetadataUpdate {
Expand Down Expand Up @@ -343,6 +363,11 @@ public Map<String, String> updated() {
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setProperties(updated);
}

@Override
public void applyTo(ViewMetadata.Builder viewMetadataBuilder) {
viewMetadataBuilder.setProperties(updated);
}
}

class RemoveProperties implements MetadataUpdate {
Expand All @@ -360,6 +385,11 @@ public Set<String> removed() {
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.removeProperties(removed);
}

@Override
public void applyTo(ViewMetadata.Builder viewMetadataBuilder) {
viewMetadataBuilder.removeProperties(removed);
}
}

class SetLocation implements MetadataUpdate {
Expand All @@ -377,5 +407,44 @@ public String location() {
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setLocation(location);
}

@Override
public void applyTo(ViewMetadata.Builder viewMetadataBuilder) {
viewMetadataBuilder.setLocation(location);
}
}

class AddViewVersion implements MetadataUpdate {
private final ViewVersion viewVersion;

public AddViewVersion(ViewVersion viewVersion) {
this.viewVersion = viewVersion;
}

public ViewVersion viewVersion() {
return viewVersion;
}

@Override
public void applyTo(ViewMetadata.Builder viewMetadataBuilder) {
viewMetadataBuilder.addVersion(viewVersion);
}
}

class SetCurrentViewVersion implements MetadataUpdate {
private final int versionId;

public SetCurrentViewVersion(int versionId) {
this.versionId = versionId;
}

public int versionId() {
return versionId;
}

@Override
public void applyTo(ViewMetadata.Builder viewMetadataBuilder) {
viewMetadataBuilder.setCurrentVersionId(versionId);
}
}
}
42 changes: 42 additions & 0 deletions core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.util.JsonUtil;
import org.apache.iceberg.view.ViewVersionParser;

public class MetadataUpdateParser {

Expand All @@ -54,6 +55,8 @@ private MetadataUpdateParser() {}
static final String SET_LOCATION = "set-location";
static final String SET_STATISTICS = "set-statistics";
static final String REMOVE_STATISTICS = "remove-statistics";
static final String ADD_VIEW_VERSION = "add-view-version";
static final String SET_CURRENT_VIEW_VERSION = "set-current-view-version";

// AssignUUID
private static final String UUID = "uuid";
Expand Down Expand Up @@ -112,6 +115,12 @@ private MetadataUpdateParser() {}
// SetLocation
private static final String LOCATION = "location";

// AddViewVersion
private static final String VIEW_VERSION = "view-version";
Copy link
Contributor

@rdblue rdblue Jul 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you update the REST spec as well?

Copy link
Contributor Author

@nastra nastra Jul 28, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was planning on updating the REST spec as part of #7913 because it requires a bunch of other View-specific stuff and it seems easier to update everything in one go


// SetCurrentViewVersion
private static final String VIEW_VERSION_ID = "view-version-id";

private static final Map<Class<? extends MetadataUpdate>, String> ACTIONS =
ImmutableMap.<Class<? extends MetadataUpdate>, String>builder()
.put(MetadataUpdate.AssignUUID.class, ASSIGN_UUID)
Expand All @@ -131,6 +140,8 @@ private MetadataUpdateParser() {}
.put(MetadataUpdate.SetProperties.class, SET_PROPERTIES)
.put(MetadataUpdate.RemoveProperties.class, REMOVE_PROPERTIES)
.put(MetadataUpdate.SetLocation.class, SET_LOCATION)
.put(MetadataUpdate.AddViewVersion.class, ADD_VIEW_VERSION)
.put(MetadataUpdate.SetCurrentViewVersion.class, SET_CURRENT_VIEW_VERSION)
.buildOrThrow();

public static String toJson(MetadataUpdate metadataUpdate) {
Expand Down Expand Up @@ -208,6 +219,13 @@ public static void toJson(MetadataUpdate metadataUpdate, JsonGenerator generator
case SET_LOCATION:
writeSetLocation((MetadataUpdate.SetLocation) metadataUpdate, generator);
break;
case ADD_VIEW_VERSION:
writeAddViewVersion((MetadataUpdate.AddViewVersion) metadataUpdate, generator);
break;
case SET_CURRENT_VIEW_VERSION:
writeSetCurrentViewVersionId(
(MetadataUpdate.SetCurrentViewVersion) metadataUpdate, generator);
break;
default:
throw new IllegalArgumentException(
String.format(
Expand Down Expand Up @@ -271,6 +289,10 @@ public static MetadataUpdate fromJson(JsonNode jsonNode) {
return readRemoveProperties(jsonNode);
case SET_LOCATION:
return readSetLocation(jsonNode);
case ADD_VIEW_VERSION:
return readAddViewVersion(jsonNode);
case SET_CURRENT_VIEW_VERSION:
return readCurrentViewVersionId(jsonNode);
default:
throw new UnsupportedOperationException(
String.format("Cannot convert metadata update action to json: %s", action));
Expand Down Expand Up @@ -384,6 +406,17 @@ private static void writeSetLocation(MetadataUpdate.SetLocation update, JsonGene
gen.writeStringField(LOCATION, update.location());
}

private static void writeAddViewVersion(
MetadataUpdate.AddViewVersion metadataUpdate, JsonGenerator gen) throws IOException {
gen.writeFieldName(VIEW_VERSION);
ViewVersionParser.toJson(metadataUpdate.viewVersion(), gen);
}

private static void writeSetCurrentViewVersionId(
MetadataUpdate.SetCurrentViewVersion metadataUpdate, JsonGenerator gen) throws IOException {
gen.writeNumberField(VIEW_VERSION_ID, metadataUpdate.versionId());
}

private static MetadataUpdate readAssignUUID(JsonNode node) {
String uuid = JsonUtil.getString(UUID, node);
return new MetadataUpdate.AssignUUID(uuid);
Expand Down Expand Up @@ -512,4 +545,13 @@ private static MetadataUpdate readSetLocation(JsonNode node) {
String location = JsonUtil.getString(LOCATION, node);
return new MetadataUpdate.SetLocation(location);
}

private static MetadataUpdate readAddViewVersion(JsonNode node) {
return new MetadataUpdate.AddViewVersion(
ViewVersionParser.fromJson(JsonUtil.get(VIEW_VERSION, node)));
}

private static MetadataUpdate readCurrentViewVersionId(JsonNode node) {
return new MetadataUpdate.SetCurrentViewVersion(JsonUtil.getInt(VIEW_VERSION_ID, node));
}
}
Loading