Skip to content

Core: Add REST catalog adapter and tests #4241

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rdblue wants to merge 3 commits into from
Closed

Core: Add REST catalog adapter and tests #4241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aff510d
Add RESTCatalogAdaptor.
rdblue Feb 25, 2022
7565468
Core: Implement REST catalog commits.
rdblue Feb 27, 2022
ce1b8b9
Add working transactions.
rdblue Mar 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions api/src/main/java/org/apache/iceberg/util/CharSequenceSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,17 @@
package org.apache.iceberg.util;

import java.io.Serializable;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.collect.Iterators;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.relocated.com.google.common.collect.Streams;

public class CharSequenceSet implements Set<CharSequence>, Serializable {
private static final ThreadLocal<CharSequenceWrapper> wrappers = ThreadLocal.withInitial(
Expand Down Expand Up @@ -152,4 +156,28 @@ public boolean removeAll(Collection<?> objects) {
public void clear() {
wrapperSet.clear();
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}

if (o == null || getClass() != o.getClass()) {
return false;
}

CharSequenceSet that = (CharSequenceSet) o;
return wrapperSet.equals(that.wrapperSet);
}

@Override
public int hashCode() {
return Objects.hash(wrapperSet);
}

@Override
public String toString() {
return "CharSequenceSet({" + Streams.stream(iterator()).collect(Collectors.joining(", ")) + "})";
Copy link
Member

@szehon-ho szehon-ho Feb 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit, Collectors.joining seems to also take prefix, suffix argument, instead of adding them manually?

Copy link
Contributor Author

@rdblue rdblue Mar 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good to know!

}
}
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/iceberg/BaseMetastoreCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public Transaction createOrReplaceTransaction() {
private Transaction newReplaceTableTransaction(boolean orCreate) {
TableOperations ops = newTableOps(identifier);
if (!orCreate && ops.current() == null) {
throw new NoSuchTableException("No such table: %s", identifier);
throw new NoSuchTableException("Table does not exist: %s", identifier);
}

TableMetadata metadata;
Expand Down
7 changes: 6 additions & 1 deletion core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.function.Function;
import java.util.function.Predicate;
import org.apache.iceberg.encryption.EncryptionManager;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.CommitFailedException;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.io.FileIO;
Expand Down Expand Up @@ -115,7 +116,11 @@ protected void doRefresh() {
public void commit(TableMetadata base, TableMetadata metadata) {
// if the metadata is already out of date, reject it
if (base != current()) {
throw new CommitFailedException("Cannot commit: stale table metadata");
if (base != null) {
throw new CommitFailedException("Cannot commit: stale table metadata");
} else {
throw new AlreadyExistsException("Table already exists: %s", tableName());
}
}
// if the metadata is not changed, return early
if (base == metadata) {
Expand Down
10 changes: 9 additions & 1 deletion core/src/main/java/org/apache/iceberg/BaseTransaction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS;
import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT;

class BaseTransaction implements Transaction {
public class BaseTransaction implements Transaction {
private static final Logger LOG = LoggerFactory.getLogger(BaseTransaction.class);

enum TransactionType {
Expand Down Expand Up @@ -90,6 +90,14 @@ public Table table() {
return transactionTable;
}

public TableMetadata startMetadata() {
return current;
}

public TableOperations underyingOps() {
return ops;
}

private void checkLastOperationCommitted(String operation) {
Preconditions.checkState(hasLastOpCommitted,
"Cannot create new %s: last operation has not committed", operation);
Expand Down
81 changes: 80 additions & 1 deletion core/src/main/java/org/apache/iceberg/MetadataUpdate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@
import java.io.Serializable;
import java.util.Map;
import java.util.Set;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;

/**
* Represents a change to table metadata.
*/
public interface MetadataUpdate extends Serializable {
void applyTo(TableMetadata.Builder metadataBuilder);

class AssignUUID implements MetadataUpdate {
private final String uuid;

Expand All @@ -37,6 +40,11 @@ public AssignUUID(String uuid) {
public String uuid() {
return uuid;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
throw new UnsupportedOperationException("Not implemented");
}
}

class UpgradeFormatVersion implements MetadataUpdate {
Expand All @@ -49,6 +57,11 @@ public UpgradeFormatVersion(int formatVersion) {
public int formatVersion() {
return formatVersion;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.upgradeFormatVersion(formatVersion);
}
}

class AddSchema implements MetadataUpdate {
Expand All @@ -67,6 +80,11 @@ public Schema schema() {
public int lastColumnId() {
return lastColumnId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addSchema(schema, lastColumnId);
}
}

class SetCurrentSchema implements MetadataUpdate {
Expand All @@ -79,6 +97,11 @@ public SetCurrentSchema(int schemaId) {
public int schemaId() {
return schemaId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setCurrentSchema(schemaId);
}
}

class AddPartitionSpec implements MetadataUpdate {
Expand All @@ -91,6 +114,11 @@ public AddPartitionSpec(PartitionSpec spec) {
public PartitionSpec spec() {
return spec;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addPartitionSpec(spec);
}
}

class SetDefaultPartitionSpec implements MetadataUpdate {
Expand All @@ -103,6 +131,11 @@ public SetDefaultPartitionSpec(int schemaId) {
public int specId() {
return specId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setDefaultPartitionSpec(specId);
}
}

class AddSortOrder implements MetadataUpdate {
Expand All @@ -112,9 +145,14 @@ public AddSortOrder(SortOrder sortOrder) {
this.sortOrder = sortOrder;
}

public SortOrder spec() {
public SortOrder sortOrder() {
return sortOrder;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addSortOrder(sortOrder);
}
}

class SetDefaultSortOrder implements MetadataUpdate {
Expand All @@ -127,6 +165,11 @@ public SetDefaultSortOrder(int sortOrderId) {
public int sortOrderId() {
return sortOrderId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setDefaultSortOrder(sortOrderId);
}
}

class AddSnapshot implements MetadataUpdate {
Expand All @@ -139,6 +182,11 @@ public AddSnapshot(Snapshot snapshot) {
public Snapshot snapshot() {
return snapshot;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addSnapshot(snapshot);
}
}

class RemoveSnapshot implements MetadataUpdate {
Expand All @@ -151,6 +199,11 @@ public RemoveSnapshot(long snapshotId) {
public long snapshotId() {
return snapshotId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.removeSnapshots(ImmutableSet.of(snapshotId));
}
}

class RemoveSnapshotRef implements MetadataUpdate {
Expand All @@ -163,6 +216,12 @@ public RemoveSnapshotRef(String name) {
public String name() {
return name;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
// TODO: this should be generalized when tagging is supported
metadataBuilder.removeBranch(name);
}
}

class SetSnapshotRef implements MetadataUpdate {
Expand All @@ -181,6 +240,11 @@ public String name() {
public long snapshotId() {
return snapshotId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setBranchSnapshot(snapshotId, name);
}
}

class SetProperties implements MetadataUpdate {
Expand All @@ -193,6 +257,11 @@ public SetProperties(Map<String, String> updated) {
public Map<String, String> updated() {
return updated;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setProperties(updated);
}
}

class RemoveProperties implements MetadataUpdate {
Expand All @@ -205,6 +274,11 @@ public RemoveProperties(Set<String> removed) {
public Set<String> removed() {
return removed;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.removeProperties(removed);
}
}

class SetLocation implements MetadataUpdate {
Expand All @@ -217,5 +291,10 @@ public SetLocation(String location) {
public String location() {
return location;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setLocation(location);
}
}
}
Loading