From b194c0bcecb23ad00a094cfc3087f50373838e25 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 3 Sep 2021 11:15:44 +0200 Subject: [PATCH 1/2] Bump Nessie to 0.15.1 + related changes * Use new NessieApiV1 * Update reference syntax to `table-identifier ( '@' reference-name )? ( '#' pointer )?` (for Nessie-Iceberg-GC) * Slightly nicer commit message from `NessieTableOperations` * Use new `TableIdGenerators` as the "global state" tracked in Nessie --- build.gradle | 2 + .../iceberg/BaseMetastoreTableOperations.java | 10 +- .../org/apache/iceberg/TableMetadata.java | 58 ++++ .../apache/iceberg/nessie/NessieCatalog.java | 123 ++++---- .../iceberg/nessie/NessieTableOperations.java | 120 ++++++-- .../org/apache/iceberg/nessie/NessieUtil.java | 22 +- .../apache/iceberg/nessie/TableReference.java | 98 ------ .../iceberg/nessie/UpdateableReference.java | 36 ++- .../iceberg/nessie/BaseTestIceberg.java | 100 ++++-- .../iceberg/nessie/TestBranchVisibility.java | 290 ++++++++++++++++-- .../iceberg/nessie/TestNessieTable.java | 180 +++++++---- .../iceberg/nessie/TestTableReference.java | 136 -------- site/mkdocs.yml | 2 +- versions.props | 2 +- 14 files changed, 706 insertions(+), 473 deletions(-) delete mode 100644 nessie/src/main/java/org/apache/iceberg/nessie/TableReference.java delete mode 100644 nessie/src/test/java/org/apache/iceberg/nessie/TestTableReference.java diff --git a/build.gradle b/build.gradle index 38388747d25b..85bfe423c846 100644 --- a/build.gradle +++ b/build.gradle @@ -555,8 +555,10 @@ project(':iceberg-nessie') { testImplementation "org.projectnessie:nessie-jaxrs-testextension" // Need to "pull in" el-api explicitly :( testImplementation "jakarta.el:jakarta.el-api" + testImplementation 'org.assertj:assertj-core' compileOnly "org.apache.hadoop:hadoop-common" + testImplementation "org.apache.avro:avro" testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') } diff --git a/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java b/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java index 10b570c8bcf7..e75e67cf2f7c 100644 --- a/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java +++ b/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java @@ -22,6 +22,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.function.Predicate; import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.exceptions.CommitFailedException; @@ -166,6 +167,12 @@ protected void refreshFromMetadataLocation(String newLocation, int numRetries) { protected void refreshFromMetadataLocation(String newLocation, Predicate shouldRetry, int numRetries) { + refreshFromMetadataLocation(newLocation, shouldRetry, numRetries, + metadataLocation -> TableMetadataParser.read(io(), metadataLocation)); + } + + protected void refreshFromMetadataLocation(String newLocation, Predicate shouldRetry, + int numRetries, Function metadataLoader) { // use null-safe equality check because new tables have a null metadata location if (!Objects.equal(currentMetadataLocation, newLocation)) { LOG.info("Refreshing table metadata from new version: {}", newLocation); @@ -175,8 +182,7 @@ protected void refreshFromMetadataLocation(String newLocation, Predicate newMetadata.set( - TableMetadataParser.read(io(), metadataLocation))); + .run(metadataLocation -> newMetadata.set(metadataLoader.apply(metadataLocation))); String newUUID = newMetadata.get().uuid(); if (currentMetadata != null && currentMetadata.uuid() != null && newUUID != null) { diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 12e4fc2e1e07..59c71d7b55a8 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -713,6 +713,64 @@ public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { snapshots, newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); } + /** + * Returns an updated {@link TableMetadata} with the current-snapshot-ID set to the given + * snapshot-ID and the snapshot-log reset to contain only the snapshot with the given snapshot-ID. + * + * @param snapshotId ID of a snapshot that must exist, or {@code -1L} to remove the current snapshot + * and return an empty snapshot log. + * @return {@link TableMetadata} with updated {@link #currentSnapshotId} and {@link #snapshotLog} + */ + public TableMetadata withCurrentSnapshotOnly(long snapshotId) { + if ((currentSnapshotId == -1L && snapshotId == -1L && snapshots.isEmpty()) || + (currentSnapshotId == snapshotId && snapshots.size() == 1)) { + return this; + } + List newSnapshotLog = Lists.newArrayList(); + if (snapshotId != -1L) { + Snapshot snapshot = snapshotsById.get(snapshotId); + Preconditions.checkArgument(snapshot != null, "Non-existent snapshot"); + newSnapshotLog.add(new SnapshotLogEntry(snapshot.timestampMillis(), snapshotId)); + } + return new TableMetadata(null, formatVersion, uuid, location, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshotId, + snapshots, newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + } + + public TableMetadata withCurrentSchema(int schemaId) { + if (currentSchemaId == schemaId) { + return this; + } + Preconditions.checkArgument(schemasById.containsKey(schemaId), "Non-existent schema"); + return new TableMetadata(null, formatVersion, uuid, location, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schemaId, schemas, defaultSpecId, specs, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + } + + public TableMetadata withDefaultSortOrder(int sortOrderId) { + if (defaultSortOrderId == sortOrderId) { + return this; + } + Preconditions.checkArgument(sortOrdersById.containsKey(sortOrderId), "Non-existent sort-order"); + return new TableMetadata(null, formatVersion, uuid, location, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, + lastAssignedPartitionId, sortOrderId, sortOrders, properties, currentSnapshotId, + snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + } + + public TableMetadata withDefaultSpec(int specId) { + if (defaultSpecId == specId) { + return this; + } + Preconditions.checkArgument(specsById.containsKey(specId), "Non-existent partition spec"); + return new TableMetadata(null, formatVersion, uuid, location, + lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, specId, specs, + lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, + snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + } + private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil.NextID nextID) { PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(partitionSpec.schema()) .withSpecId(partitionSpec.specId()); diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java index a6e5e5f1a5b0..e0bfb122fd14 100644 --- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java +++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java @@ -41,25 +41,27 @@ import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.hadoop.HadoopFileIO; import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Joiner; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.util.Tasks; -import org.projectnessie.api.TreeApi; -import org.projectnessie.api.params.EntriesParams; -import org.projectnessie.client.NessieClient; import org.projectnessie.client.NessieConfigConstants; +import org.projectnessie.client.api.CommitMultipleOperationsBuilder; +import org.projectnessie.client.api.NessieApiV1; +import org.projectnessie.client.http.HttpClientBuilder; import org.projectnessie.client.http.HttpClientException; import org.projectnessie.error.BaseNessieClientServerException; import org.projectnessie.error.NessieConflictException; import org.projectnessie.error.NessieNotFoundException; import org.projectnessie.model.Branch; -import org.projectnessie.model.Contents; +import org.projectnessie.model.Content; +import org.projectnessie.model.ContentKey; import org.projectnessie.model.IcebergTable; -import org.projectnessie.model.ImmutableDelete; -import org.projectnessie.model.ImmutableOperations; -import org.projectnessie.model.ImmutablePut; -import org.projectnessie.model.Operations; +import org.projectnessie.model.Operation; import org.projectnessie.model.Reference; +import org.projectnessie.model.TableReference; +import org.projectnessie.model.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +77,7 @@ public class NessieCatalog extends BaseMetastoreCatalog implements AutoCloseable, SupportsNamespaces, Configurable { private static final Logger logger = LoggerFactory.getLogger(NessieCatalog.class); private static final Joiner SLASH = Joiner.on("/"); - private NessieClient client; + private NessieApiV1 api; private String warehouseLocation; private Configuration config; private UpdateableReference reference; @@ -95,7 +97,8 @@ public void initialize(String inputName, Map options) { // remove nessie prefix final Function removePrefix = x -> x.replace("nessie.", ""); - this.client = NessieClient.builder().fromConfig(x -> options.get(removePrefix.apply(x))).build(); + this.api = HttpClientBuilder.builder().fromConfig(x -> options.get(removePrefix.apply(x))) + .build(NessieApiV1.class); this.warehouseLocation = options.get(CatalogProperties.WAREHOUSE_LOCATION); if (warehouseLocation == null) { @@ -120,12 +123,12 @@ public void initialize(String inputName, Map options) { throw new IllegalStateException("Parameter 'warehouse' not set, Nessie can't store data."); } final String requestedRef = options.get(removePrefix.apply(NessieConfigConstants.CONF_NESSIE_REF)); - this.reference = loadReference(requestedRef); + this.reference = loadReference(requestedRef, null); } @Override public void close() { - client.close(); + api.close(); } @Override @@ -135,15 +138,17 @@ public String name() { @Override protected TableOperations newTableOps(TableIdentifier tableIdentifier) { - TableReference pti = TableReference.parse(tableIdentifier); + TableReference tr = TableReference.parse(tableIdentifier.name()); + Preconditions.checkArgument(!tr.hasTimestamp(), "Invalid table name: # is only allowed for hashes (reference by " + + "timestamp is not supported)"); UpdateableReference newReference = this.reference; - if (pti.reference() != null) { - newReference = loadReference(pti.reference()); + if (tr.getReference() != null) { + newReference = loadReference(tr.getReference(), tr.getHash()); } return new NessieTableOperations( - NessieUtil.toKey(pti.tableIdentifier()), + ContentKey.of(org.projectnessie.model.Namespace.of(tableIdentifier.namespace().levels()), tr.getName()), newReference, - client, + api, fileIO, catalogOptions); } @@ -170,23 +175,27 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { return false; } - Operations contents = ImmutableOperations.builder() - .addOperations(ImmutableDelete.builder().key(NessieUtil.toKey(identifier)).build()) - .commitMeta(NessieUtil.buildCommitMetadata(String.format("iceberg delete table '%s'", identifier), + if (purge) { + logger.info("Purging data for table {} was set to true but is ignored", identifier.toString()); + } + + CommitMultipleOperationsBuilder commitBuilderBase = api.commitMultipleOperations() + .commitMeta(NessieUtil.buildCommitMetadata(String.format("Iceberg delete table %s", identifier), catalogOptions)) - .build(); + .operation(Operation.Delete.of(NessieUtil.toKey(identifier))); // We try to drop the table. Simple retry after ref update. boolean threw = true; try { - Tasks.foreach(contents) + Tasks.foreach(commitBuilderBase) .retry(5) .stopRetryOn(NessieNotFoundException.class) .throwFailureWhenFinished() - .onFailure((c, exception) -> refresh()) - .run(c -> { - Branch branch = client.getTreeApi().commitMultipleOperations(reference.getAsBranch().getName(), - reference.getHash(), c); + .onFailure((o, exception) -> refresh()) + .run(commitBuilder -> { + Branch branch = commitBuilder + .branch(reference.getAsBranch()) + .commit(); reference.updateReference(branch); }, BaseNessieClientServerException.class); threw = false; @@ -215,24 +224,22 @@ public void renameTable(TableIdentifier from, TableIdentifier toOriginal) { throw new AlreadyExistsException("table %s already exists", to.name()); } - Operations contents = ImmutableOperations.builder() - .addOperations( - ImmutablePut.builder().key(NessieUtil.toKey(to)).contents(existingFromTable).build(), - ImmutableDelete.builder().key(NessieUtil.toKey(from)).build()) - .commitMeta(NessieUtil.buildCommitMetadata(String.format("iceberg rename table from '%s' to '%s'", - from, to), - catalogOptions)) - .build(); + CommitMultipleOperationsBuilder operations = api.commitMultipleOperations() + .commitMeta(NessieUtil.buildCommitMetadata(String.format("Iceberg rename table from '%s' to '%s'", + from, to), catalogOptions)) + .operation(Operation.Put.of(NessieUtil.toKey(to), existingFromTable, existingFromTable)) + .operation(Operation.Delete.of(NessieUtil.toKey(from))); try { - Tasks.foreach(contents) + Tasks.foreach(operations) .retry(5) .stopRetryOn(NessieNotFoundException.class) .throwFailureWhenFinished() - .onFailure((c, exception) -> refresh()) - .run(c -> { - Branch branch = client.getTreeApi().commitMultipleOperations(reference.getAsBranch().getName(), - reference.getHash(), c); + .onFailure((o, exception) -> refresh()) + .run(ops -> { + Branch branch = ops + .branch(reference.getAsBranch()) + .commit(); reference.updateReference(branch); }, BaseNessieClientServerException.class); } catch (NessieNotFoundException e) { @@ -322,37 +329,46 @@ public Configuration getConf() { return config; } - TreeApi getTreeApi() { - return client.getTreeApi(); - } - public void refresh() throws NessieNotFoundException { - reference.refresh(); + reference.refresh(api); } public String currentHash() { return reference.getHash(); } + @VisibleForTesting String currentRefName() { return reference.getName(); } + @VisibleForTesting + FileIO fileIO() { + return fileIO; + } + private IcebergTable table(TableIdentifier tableIdentifier) { try { - Contents table = client.getContentsApi() - .getContents(NessieUtil.toKey(tableIdentifier), reference.getName(), reference.getHash()); - return table.unwrap(IcebergTable.class).orElse(null); + ContentKey key = NessieUtil.toKey(tableIdentifier); + Content table = api.getContent().key(key).reference(reference.getReference()).get().get(key); + return table != null ? table.unwrap(IcebergTable.class).orElse(null) : null; } catch (NessieNotFoundException e) { return null; } } - private UpdateableReference loadReference(String requestedRef) { + private UpdateableReference loadReference(String requestedRef, String hash) { try { - Reference ref = requestedRef == null ? client.getTreeApi().getDefaultBranch() - : client.getTreeApi().getReferenceByName(requestedRef); - return new UpdateableReference(ref, client.getTreeApi()); + Reference ref = requestedRef == null ? api.getDefaultBranch() + : api.getReference().refName(requestedRef).get(); + if (hash != null) { + if (ref instanceof Branch) { + ref = Branch.of(ref.getName(), hash); + } else { + ref = Tag.of(ref.getName(), hash); + } + } + return new UpdateableReference(ref, hash != null); } catch (NessieNotFoundException ex) { if (requestedRef != null) { throw new IllegalArgumentException(String.format( @@ -369,8 +385,9 @@ private UpdateableReference loadReference(String requestedRef) { private Stream tableStream(Namespace namespace) { try { - return client.getTreeApi() - .getEntries(reference.getName(), EntriesParams.builder().hashOnRef(reference.getHash()).build()) + return api.getEntries() + .reference(reference.getReference()) + .get() .getEntries() .stream() .filter(NessieUtil.namespacePredicate(namespace)) diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java index 27a1d1c8500e..106cad40cab6 100644 --- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java +++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java @@ -20,49 +20,55 @@ package org.apache.iceberg.nessie; import java.util.Map; +import java.util.function.Predicate; import org.apache.iceberg.BaseMetastoreTableOperations; +import org.apache.iceberg.Snapshot; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.CommitStateUnknownException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.io.FileIO; -import org.projectnessie.client.NessieClient; +import org.projectnessie.client.api.NessieApiV1; import org.projectnessie.client.http.HttpClientException; import org.projectnessie.error.NessieConflictException; import org.projectnessie.error.NessieNotFoundException; import org.projectnessie.model.Branch; -import org.projectnessie.model.Contents; -import org.projectnessie.model.ContentsKey; +import org.projectnessie.model.Content; +import org.projectnessie.model.ContentKey; import org.projectnessie.model.IcebergTable; +import org.projectnessie.model.ImmutableCommitMeta; import org.projectnessie.model.ImmutableIcebergTable; -import org.projectnessie.model.ImmutableOperations; import org.projectnessie.model.Operation; -import org.projectnessie.model.Operations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Nessie implementation of Iceberg TableOperations. */ public class NessieTableOperations extends BaseMetastoreTableOperations { - private final NessieClient client; - private final ContentsKey key; - private UpdateableReference reference; + private static final Logger LOG = LoggerFactory.getLogger(NessieTableOperations.class); + + private final NessieApiV1 api; + private final ContentKey key; + private final UpdateableReference reference; private IcebergTable table; - private FileIO fileIO; - private Map catalogOptions; + private final FileIO fileIO; + private final Map catalogOptions; /** * Create a nessie table operations given a table identifier. */ NessieTableOperations( - ContentsKey key, + ContentKey key, UpdateableReference reference, - NessieClient client, + NessieApiV1 api, FileIO fileIO, Map catalogOptions) { this.key = key; this.reference = reference; - this.client = client; + this.api = api; this.fileIO = fileIO; this.catalogOptions = catalogOptions; } @@ -72,21 +78,44 @@ protected String tableName() { return key.toString(); } + @Override + protected void refreshFromMetadataLocation(String newLocation, Predicate shouldRetry, + int numRetries) { + super.refreshFromMetadataLocation(newLocation, shouldRetry, numRetries, this::loadTableMetadata); + } + + private TableMetadata loadTableMetadata(String metadataLocation) { + // Update the TableMetadata with the Content of NessieTableState. + return TableMetadataParser.read(io(), metadataLocation) + .withCurrentSnapshotOnly(table.getSnapshotId()) + .withCurrentSchema(table.getSchemaId()) + .withDefaultSortOrder(table.getSortOrderId()) + .withDefaultSpec(table.getSpecId()); + } + @Override protected void doRefresh() { try { - reference.refresh(); + reference.refresh(api); } catch (NessieNotFoundException e) { throw new RuntimeException("Failed to refresh as ref is no longer valid.", e); } String metadataLocation = null; try { - Contents contents = client.getContentsApi().getContents(key, reference.getName(), reference.getHash()); - this.table = contents.unwrap(IcebergTable.class) - .orElseThrow(() -> - new IllegalStateException("Cannot refresh iceberg table: " + - String.format("Nessie points to a non-Iceberg object for path: %s.", key))); - metadataLocation = table.getMetadataLocation(); + Content content = api.getContent().key(key).reference(reference.getReference()).get() + .get(key); + LOG.debug("Content '{}' at '{}': {}", key, reference.getReference(), content); + if (content == null) { + if (currentMetadataLocation() != null) { + throw new NoSuchTableException("No such table %s in %s", key, reference.getReference()); + } + } else { + this.table = content.unwrap(IcebergTable.class) + .orElseThrow(() -> + new IllegalStateException("Cannot refresh iceberg table: " + + String.format("Nessie points to a non-Iceberg object for path: %s.", key))); + metadataLocation = table.getMetadataLocation(); + } } catch (NessieNotFoundException ex) { if (currentMetadataLocation() != null) { throw new NoSuchTableException(ex, "No such table %s", key); @@ -103,18 +132,31 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) { boolean delete = true; try { - ImmutableIcebergTable.Builder newTable = ImmutableIcebergTable.builder(); + ImmutableIcebergTable.Builder newTableBuilder = ImmutableIcebergTable.builder(); if (table != null) { - newTable.from(table); + newTableBuilder.id(table.getId()); } - newTable.metadataLocation(newMetadataLocation); - - ImmutableIcebergTable icebergTable = newTable.build(); - Operations op = ImmutableOperations.builder().addOperations(Operation.Put.of(key, icebergTable)) - .commitMeta(NessieUtil.buildCommitMetadata(String.format("iceberg add table '%s'", key), - catalogOptions)).build(); - Branch branch = client.getTreeApi().commitMultipleOperations(reference.getAsBranch().getName(), - reference.getHash(), op); + Snapshot snapshot = metadata.currentSnapshot(); + long snapshotId = snapshot != null ? snapshot.snapshotId() : -1L; + IcebergTable newTable = newTableBuilder + .snapshotId(snapshotId) + .schemaId(metadata.currentSchemaId()) + .specId(metadata.defaultSpecId()) + .sortOrderId(metadata.defaultSortOrderId()) + .metadataLocation(newMetadataLocation) + .build(); + + LOG.debug("Committing '{}' against '{}': {}", key, reference.getReference(), newTable); + ImmutableCommitMeta.Builder builder = ImmutableCommitMeta.builder(); + builder.message(buildCommitMsg(base, metadata)); + if (isSnapshotOperation(base, metadata)) { + builder.putProperties("iceberg.operation", snapshot.operation()); + } + Branch branch = api.commitMultipleOperations() + .operation(Operation.Put.of(key, newTable, table)) + .commitMeta(NessieUtil.catalogOptions(builder, catalogOptions).build()) + .branch(reference.getAsBranch()) + .commit(); reference.updateReference(branch); delete = false; @@ -129,7 +171,8 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) { delete = false; throw new CommitStateUnknownException(ex); } catch (NessieNotFoundException ex) { - throw new RuntimeException(String.format("Commit failed: Reference %s no longer exist", reference.getName()), ex); + throw new RuntimeException( + String.format("Commit failed: Reference %s no longer exist", reference.getName()), ex); } finally { if (delete) { io().deleteFile(newMetadataLocation); @@ -137,6 +180,21 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) { } } + private boolean isSnapshotOperation(TableMetadata base, TableMetadata metadata) { + Snapshot snapshot = metadata.currentSnapshot(); + return snapshot != null && (base == null || base.currentSnapshot() == null || + snapshot.snapshotId() != base.currentSnapshot().snapshotId()); + } + + private String buildCommitMsg(TableMetadata base, TableMetadata metadata) { + if (isSnapshotOperation(base, metadata)) { + return String.format("Iceberg %s against %s", metadata.currentSnapshot().operation(), tableName()); + } else if (base != null && metadata.currentSchemaId() != base.currentSchemaId()) { + return String.format("Iceberg schema change against %s", tableName()); + } + return String.format("Iceberg commit against %s", tableName()); + } + @Override public FileIO io() { return fileIO; diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java index 3141b75deffe..88850ce51bf6 100644 --- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java +++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java @@ -31,7 +31,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.projectnessie.model.CommitMeta; -import org.projectnessie.model.ContentsKey; +import org.projectnessie.model.ContentKey; import org.projectnessie.model.EntriesResponse; import org.projectnessie.model.ImmutableCommitMeta; @@ -80,27 +80,29 @@ static TableIdentifier removeCatalogName(TableIdentifier to, String name) { return to; } - static ContentsKey toKey(TableIdentifier tableIdentifier) { + static ContentKey toKey(TableIdentifier tableIdentifier) { List identifiers = new ArrayList<>(); if (tableIdentifier.hasNamespace()) { identifiers.addAll(Arrays.asList(tableIdentifier.namespace().levels())); } identifiers.add(tableIdentifier.name()); - ContentsKey key = ContentsKey.of(identifiers); - return key; + return ContentKey.of(identifiers); } static CommitMeta buildCommitMetadata(String commitMsg, Map catalogOptions) { + return catalogOptions(CommitMeta.builder().message(commitMsg), catalogOptions).build(); + } + + static ImmutableCommitMeta.Builder catalogOptions(ImmutableCommitMeta.Builder commitMetaBuilder, + Map catalogOptions) { Preconditions.checkArgument(null != catalogOptions, "catalogOptions must not be null"); - ImmutableCommitMeta.Builder cm = CommitMeta.builder().message(commitMsg) - .author(NessieUtil.commitAuthor(catalogOptions)); - cm.putProperties(APPLICATION_TYPE, "iceberg"); + commitMetaBuilder.author(NessieUtil.commitAuthor(catalogOptions)); + commitMetaBuilder.putProperties(APPLICATION_TYPE, "iceberg"); if (catalogOptions.containsKey(CatalogProperties.APP_ID)) { - cm.putProperties(CatalogProperties.APP_ID, catalogOptions.get(CatalogProperties.APP_ID)); + commitMetaBuilder.putProperties(CatalogProperties.APP_ID, catalogOptions.get(CatalogProperties.APP_ID)); } - - return cm.build(); + return commitMetaBuilder; } /** diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/TableReference.java b/nessie/src/main/java/org/apache/iceberg/nessie/TableReference.java deleted file mode 100644 index 41064968fbc8..000000000000 --- a/nessie/src/main/java/org/apache/iceberg/nessie/TableReference.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.nessie; - -import java.time.Instant; -import java.util.List; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.relocated.com.google.common.base.Splitter; - -public class TableReference { - - private static final Splitter BRANCH_NAME_SPLITTER = Splitter.on("@"); - private final TableIdentifier tableIdentifier; - private final Instant timestamp; - private final String reference; - - /** - * Container class to specify a TableIdentifier on a specific Reference or at an Instant in time. - */ - public TableReference(TableIdentifier tableIdentifier, Instant timestamp, String reference) { - this.tableIdentifier = tableIdentifier; - this.timestamp = timestamp; - this.reference = reference; - } - - public TableIdentifier tableIdentifier() { - return tableIdentifier; - } - - public Instant timestamp() { - return timestamp; - } - - public String reference() { - return reference; - } - - /** - * Convert dataset read/write options to a table and ref/hash. - */ - public static TableReference parse(TableIdentifier path) { - TableReference pti = parse(path.name()); - return new TableReference( - TableIdentifier.of(path.namespace(), pti.tableIdentifier().name()), - pti.timestamp(), - pti.reference()); - } - - /** - * Convert dataset read/write options to a table and ref/hash. - */ - public static TableReference parse(String path) { - // I am assuming tables can't have @ or # symbols - if (path.split("@").length > 2) { - throw new IllegalArgumentException(String.format("Can only reference one branch in %s", path)); - } - if (path.split("#").length > 2) { - throw new IllegalArgumentException(String.format("Can only reference one timestamp in %s", path)); - } - - if (path.contains("@") && path.contains("#")) { - throw new IllegalArgumentException("Invalid table name:" + - " # is not allowed (reference by timestamp is not supported)"); - } - - if (path.contains("@")) { - List tableRef = BRANCH_NAME_SPLITTER.splitToList(path); - TableIdentifier identifier = TableIdentifier.parse(tableRef.get(0)); - return new TableReference(identifier, null, tableRef.get(1)); - } - - if (path.contains("#")) { - throw new IllegalArgumentException("Invalid table name:" + - " # is not allowed (reference by timestamp is not supported)"); - } - - TableIdentifier identifier = TableIdentifier.parse(path); - - return new TableReference(identifier, null, null); - } -} diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/UpdateableReference.java b/nessie/src/main/java/org/apache/iceberg/nessie/UpdateableReference.java index eae05a53e1f7..d1b3e6c3ab8e 100644 --- a/nessie/src/main/java/org/apache/iceberg/nessie/UpdateableReference.java +++ b/nessie/src/main/java/org/apache/iceberg/nessie/UpdateableReference.java @@ -20,44 +20,44 @@ package org.apache.iceberg.nessie; import java.util.Objects; -import org.projectnessie.api.TreeApi; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.projectnessie.client.api.NessieApiV1; import org.projectnessie.error.NessieNotFoundException; import org.projectnessie.model.Branch; -import org.projectnessie.model.Hash; import org.projectnessie.model.Reference; class UpdateableReference { private Reference reference; - private final TreeApi client; + private final boolean mutable; - UpdateableReference(Reference reference, TreeApi client) { + /** + * Construct a new {@link UpdateableReference} using a Nessie reference object and a flag + * whether an explicit hash was used to create the reference object. + */ + UpdateableReference(Reference reference, boolean hashReference) { this.reference = reference; - this.client = client; + this.mutable = reference instanceof Branch && !hashReference; } - public boolean refresh() throws NessieNotFoundException { - if (reference instanceof Hash) { + public boolean refresh(NessieApiV1 api) throws NessieNotFoundException { + if (!mutable) { return false; } Reference oldReference = reference; - reference = client.getReferenceByName(reference.getName()); + reference = api.getReference().refName(reference.getName()).get(); return !oldReference.equals(reference); } public void updateReference(Reference ref) { - Objects.requireNonNull(ref); - this.reference = ref; + Preconditions.checkState(mutable, "Hash references cannot be updated."); + this.reference = Objects.requireNonNull(ref); } public boolean isBranch() { return reference instanceof Branch; } - public UpdateableReference copy() { - return new UpdateableReference(reference, client); - } - public String getHash() { return reference.getHash(); } @@ -69,10 +69,12 @@ public Branch getAsBranch() { return (Branch) reference; } + public Reference getReference() { + return reference; + } + public void checkMutable() { - if (!isBranch()) { - throw new IllegalArgumentException("You can only mutate tables when using a branch."); - } + Preconditions.checkArgument(mutable, "You can only mutate tables when using a branch without a hash or timestamp."); } public String getName() { diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java index 1907fae32748..738bd1d8df91 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java @@ -23,37 +23,57 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import org.apache.avro.generic.GenericData.Record; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.Files; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.avro.Avro; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.LongType; import org.apache.iceberg.types.Types.StructType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; -import org.projectnessie.api.ContentsApi; -import org.projectnessie.api.TreeApi; -import org.projectnessie.client.NessieClient; +import org.projectnessie.client.api.NessieApiV1; +import org.projectnessie.client.http.HttpClientBuilder; import org.projectnessie.error.NessieConflictException; import org.projectnessie.error.NessieNotFoundException; -import org.projectnessie.jaxrs.NessieJaxRsExtension; +import org.projectnessie.jaxrs.ext.NessieJaxRsExtension; import org.projectnessie.model.Branch; import org.projectnessie.model.Reference; +import org.projectnessie.model.Tag; +import org.projectnessie.versioned.persist.adapter.DatabaseAdapter; +import org.projectnessie.versioned.persist.inmem.InmemoryDatabaseAdapterFactory; +import org.projectnessie.versioned.persist.inmem.InmemoryTestConnectionProviderSource; +import org.projectnessie.versioned.persist.tests.extension.DatabaseAdapterExtension; +import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapter; +import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapterName; +import org.projectnessie.versioned.persist.tests.extension.NessieExternalDatabase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.iceberg.types.Types.NestedField.required; +@ExtendWith(DatabaseAdapterExtension.class) +@NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME) +@NessieExternalDatabase(InmemoryTestConnectionProviderSource.class) public abstract class BaseTestIceberg { + + @NessieDbAdapter + static DatabaseAdapter databaseAdapter; @RegisterExtension - static NessieJaxRsExtension server = new NessieJaxRsExtension(); + static NessieJaxRsExtension server = new NessieJaxRsExtension(() -> databaseAdapter); private static final Logger LOGGER = LoggerFactory.getLogger(BaseTestIceberg.class); @@ -61,9 +81,7 @@ public abstract class BaseTestIceberg { public Path temp; protected NessieCatalog catalog; - protected NessieClient client; - protected TreeApi tree; - protected ContentsApi contents; + protected NessieApiV1 api; protected Configuration hadoopConfig; protected final String branch; private String uri; @@ -73,30 +91,27 @@ public BaseTestIceberg(String branch) { } private void resetData() throws NessieConflictException, NessieNotFoundException { - for (Reference r : tree.getAllReferences()) { + for (Reference r : api.getAllReferences().get().getReferences()) { if (r instanceof Branch) { - tree.deleteBranch(r.getName(), r.getHash()); + api.deleteBranch().branch((Branch) r).delete(); } else { - tree.deleteTag(r.getName(), r.getHash()); + api.deleteTag().tag((Tag) r).delete(); } } - tree.createReference(Branch.of("main", null)); + api.createReference().reference(Branch.of("main", null)).create(); } @BeforeEach public void beforeEach() throws IOException { - String port = System.getProperty("quarkus.http.test-port", "19120"); uri = server.getURI().toString(); - this.client = NessieClient.builder().withUri(uri).build(); - tree = client.getTreeApi(); - contents = client.getContentsApi(); + this.api = HttpClientBuilder.builder().withUri(uri).build(NessieApiV1.class); resetData(); try { - tree.createReference(Branch.of(branch, null)); + api.createReference().reference(Branch.of(branch, null)).create(); } catch (Exception e) { - // ignore, already created. Cant run this in BeforeAll as quarkus hasn't disabled auth + // ignore, already created. Can't run this in BeforeAll as quarkus hasn't disabled auth } hadoopConfig = new Configuration(); @@ -136,17 +151,28 @@ protected static Schema schema(int count) { return new Schema(Types.StructType.of(fields).fields()); } - void createBranch(String name, String hash) throws NessieNotFoundException, NessieConflictException { - tree.createReference(Branch.of(name, hash)); + void createBranch(String name, String hash) + throws NessieNotFoundException, NessieConflictException { + createBranch(name, hash, "main"); + } + + void createBranch(String name, String hash, String sourceRef) + throws NessieNotFoundException, NessieConflictException { + api.createReference().reference(Branch.of(name, hash)).sourceRefName(sourceRef).create(); } @AfterEach public void afterEach() throws Exception { - catalog.close(); - client.close(); - catalog = null; - client = null; - hadoopConfig = null; + try { + if (catalog != null) { + catalog.close(); + } + api.close(); + } finally { + catalog = null; + api = null; + hadoopConfig = null; + } } static String metadataLocation(NessieCatalog catalog, TableIdentifier tableIdentifier) { @@ -156,4 +182,28 @@ static String metadataLocation(NessieCatalog catalog, TableIdentifier tableIdent NessieTableOperations icebergOps = (NessieTableOperations) ops; return icebergOps.currentMetadataLocation(); } + + static String writeRecordsToFile(Table table, Schema schema, String filename, + List records) + throws IOException { + String fileLocation = table.location().replace("file:", "") + + String.format("/data/%s.avro", filename); + try (FileAppender writer = Avro.write(Files.localOutput(fileLocation)) + .schema(schema) + .named("test") + .build()) { + for (Record rec : records) { + writer.add(rec); + } + } + return fileLocation; + } + + static DataFile makeDataFile(Table icebergTable, String fileLocation) { + return DataFiles.builder(icebergTable.spec()) + .withRecordCount(3) + .withPath(fileLocation) + .withFileSizeInBytes(Files.localInput(fileLocation).getLength()) + .build(); + } } diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java index 26d7d8807b23..64bd6660be31 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java @@ -19,14 +19,33 @@ package org.apache.iceberg.nessie; +import java.util.Collections; +import java.util.Map; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.Transaction; +import org.apache.iceberg.avro.AvroSchemaUtil; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.types.Types.NestedField; +import org.assertj.core.api.AbstractStringAssert; import org.assertj.core.api.Assertions; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.projectnessie.error.NessieConflictException; import org.projectnessie.error.NessieNotFoundException; +import org.projectnessie.model.Branch; +import org.projectnessie.model.ContentKey; +import org.projectnessie.model.IcebergTable; +import org.projectnessie.model.Reference; public class TestBranchVisibility extends BaseTestIceberg { @@ -53,29 +72,27 @@ public void after() throws NessieNotFoundException, NessieConflictException { catalog.dropTable(tableIdentifier1); catalog.dropTable(tableIdentifier2); catalog.refresh(); - catalog.getTreeApi().deleteBranch("test", catalog.getTreeApi().getReferenceByName("test").getHash()); + for (Reference reference : api.getAllReferences().get().getReferences()) { + if (!reference.getName().equals("main")) { + api.deleteBranch().branch((Branch) reference).delete(); + } + } testCatalog = null; } @Test public void testBranchNoChange() { - testCatalogEquality(catalog, testCatalog, true, true); + testCatalogEquality(catalog, testCatalog, true, true, () -> { }); } + /** Ensure catalogs can't see each others updates. */ @Test public void testUpdateCatalogs() { - // ensure catalogs can't see each others updates - updateSchema(catalog, tableIdentifier1); - - testCatalogEquality(catalog, testCatalog, false, true); - - String initialMetadataLocation = metadataLocation(testCatalog, tableIdentifier2); - updateSchema(testCatalog, tableIdentifier2); + testCatalogEquality(catalog, testCatalog, false, true, + () -> updateSchema(catalog, tableIdentifier1)); - testCatalogEquality(catalog, testCatalog, false, false); - - // points to the previous metadata location - Assertions.assertThat(initialMetadataLocation).isEqualTo(metadataLocation(catalog, tableIdentifier2)); + testCatalogEquality(catalog, testCatalog, true, false, + () -> updateSchema(catalog, tableIdentifier2)); } @Test @@ -85,11 +102,11 @@ public void testCatalogOnReference() { // catalog created with ref points to same catalog as above NessieCatalog refCatalog = initCatalog("test"); - testCatalogEquality(refCatalog, testCatalog, true, true); + testCatalogEquality(refCatalog, testCatalog, true, true, () -> { }); // catalog created with hash points to same catalog as above NessieCatalog refHashCatalog = initCatalog("main"); - testCatalogEquality(refHashCatalog, catalog, true, true); + testCatalogEquality(refHashCatalog, catalog, true, true, () -> { }); } @Test @@ -118,35 +135,244 @@ public void testConcurrentChanges() { updateSchema(emptyTestCatalog, tableIdentifier1); } + @Test + public void testSchemaSnapshot() throws Exception { + + String branchTest = "test"; + String branch1 = "branch-1"; + String branch2 = "branch-2"; + + NessieCatalog catalog = initCatalog(branchTest); + String metadataOnTest = addRow(catalog, tableIdentifier1, "initial-data", + ImmutableMap.of("id0", 4L)); + long snapshotIdOnTest = snapshotIdFromMetadata(catalog, metadataOnTest); + catalog.refresh(); + + String hashOnTest = catalog.currentHash(); + createBranch(branch1, hashOnTest, branchTest); + createBranch(branch2, hashOnTest, branchTest); + + String metadataOnTest2 = addRow(catalog, tableIdentifier1, "added-data-on-test", + ImmutableMap.of("id0", 5L)); + Assertions.assertThat(metadataOnTest2).isNotEqualTo(metadataOnTest); + long snapshotIdOnTest2 = snapshotIdFromMetadata(catalog, metadataOnTest2); + verifyRefState(catalog, tableIdentifier1, snapshotIdOnTest2, 0); + + NessieCatalog catalogBranch1 = initCatalog(branch1); + updateSchema(catalogBranch1, tableIdentifier1, Types.StringType.get()); + verifyRefState(catalogBranch1, tableIdentifier1, snapshotIdOnTest, 1); + String metadataOn1 = addRow(catalogBranch1, tableIdentifier1, "testSchemaSnapshot-in-1", + ImmutableMap.of("id0", 42L, "id1", "world")); + Assertions.assertThat(metadataOn1).isNotEqualTo(metadataOnTest).isNotEqualTo(metadataOnTest2); + + NessieCatalog catalogBranch2 = initCatalog(branch2); + updateSchema(catalogBranch2, tableIdentifier1, Types.IntegerType.get()); + verifyRefState(catalogBranch2, tableIdentifier1, snapshotIdOnTest, 2); + String metadataOn2 = addRow(catalogBranch2, tableIdentifier1, "testSchemaSnapshot-in-2", + ImmutableMap.of("id0", 43L, "id2", 666)); + Assertions.assertThat(metadataOn2).isNotEqualTo(metadataOnTest).isNotEqualTo(metadataOnTest2); + } + + /** + * Complex-ish test case that verifies that both the snapshot-ID and schema-ID are properly set + * and retained when working with a mixture of DDLs and DMLs across multiple branches. + */ + @Test + public void testStateTrackingOnMultipleBranches() throws Exception { + + String branchTest = "test"; + String branchA = "branch_a"; + String branchB = "branch_b"; + + NessieCatalog catalog = initCatalog(branchTest); + verifySchema(catalog, tableIdentifier1, Types.LongType.get()); + String initialLocation = metadataLocation(catalog, tableIdentifier1); + + // Verify last-column-id + verifyRefState(catalog, tableIdentifier1, -1L, 0); + + // Add a row and verify that the + String metadataOnTest = addRow(catalog, tableIdentifier1, "initial-data", + Collections.singletonMap("id0", 1L)); + Assertions.assertThat(metadataOnTest).isNotEqualTo(initialLocation); + long snapshotIdOnTest = snapshotIdFromMetadata(catalog, metadataOnTest); + verifyRefState(catalog, tableIdentifier1, snapshotIdOnTest, 0); + + String hashOnTest = catalog.currentHash(); + createBranch(branchA, hashOnTest, branchTest); + createBranch(branchB, hashOnTest, branchTest); + + NessieCatalog catalogBranchA = initCatalog(branchA); + // branchA hasn't been modified yet, so it must be "equal" to branch "test" + verifyRefState(catalogBranchA, tableIdentifier1, snapshotIdOnTest, 0); + // updateSchema updates the schema on branch "branch_a", but not on branch "test" + updateSchema(catalogBranchA, tableIdentifier1, Types.StringType.get()); + verifyRefState(catalogBranchA, tableIdentifier1, snapshotIdOnTest, 1); + verifySchema(catalogBranchA, tableIdentifier1, Types.LongType.get(), Types.StringType.get()); + verifyRefState(catalog, tableIdentifier1, snapshotIdOnTest, 0); + + String metadataOnA1 = addRow(catalogBranchA, tableIdentifier1, "branch-a-1", + ImmutableMap.of("id0", 2L, "id1", "hello")); + // addRow() must produce a new metadata + Assertions.assertThat(metadataOnA1).isNotEqualTo(metadataOnTest); + long snapshotIdOnA1 = snapshotIdFromMetadata(catalogBranchA, metadataOnA1); + Assertions.assertThat(snapshotIdOnA1).isNotEqualTo(snapshotIdOnTest); + verifyRefState(catalogBranchA, tableIdentifier1, snapshotIdOnA1, 1); + verifyRefState(catalog, tableIdentifier1, snapshotIdOnTest, 0); + + NessieCatalog catalogBranchB = initCatalog(branchB); + long snapshotIdOnB = snapshotIdFromNessie(catalogBranchB, tableIdentifier1); + Assertions.assertThat(snapshotIdOnB).isEqualTo(snapshotIdOnTest); + // branchB hasn't been modified yet, so it must be "equal" to branch "test" + verifyRefState(catalogBranchB, tableIdentifier1, snapshotIdOnB, 0); + // updateSchema should use schema-id 2 because schema-id 1 has already been used by the above + // schema change in branch_a. + updateSchema(catalogBranchB, tableIdentifier1, Types.LongType.get()); + verifyRefState(catalogBranchB, tableIdentifier1, snapshotIdOnB, 2); + verifySchema(catalogBranchB, tableIdentifier1, Types.LongType.get(), Types.LongType.get()); + verifyRefState(catalog, tableIdentifier1, snapshotIdOnTest, 0); + + String metadataOnB1 = addRow(catalogBranchB, tableIdentifier1, "branch-b-1", + ImmutableMap.of("id0", 3L, "id2", 42L)); + long snapshotIdOnB1 = snapshotIdFromMetadata(catalogBranchB, metadataOnB1); + // addRow() must produce a new metadata + Assertions.assertThat(metadataOnB1) + .isNotEqualTo(metadataOnA1) + .isNotEqualTo(metadataOnTest); + verifyRefState(catalogBranchB, tableIdentifier1, snapshotIdOnB1, 2); + verifyRefState(catalog, tableIdentifier1, snapshotIdOnTest, 0); + + // repeat addRow() against branchA + catalogBranchA = initCatalog(branchA); + verifySchema(catalogBranchA, tableIdentifier1, Types.LongType.get(), Types.StringType.get()); + String metadataOnA2 = addRow(catalogBranchA, tableIdentifier1, "branch-a-2", + ImmutableMap.of("id0", 4L, "id1", "hello")); + long snapshotIdOnA2 = snapshotIdFromMetadata(catalogBranchA, metadataOnA2); + Assertions.assertThat(metadataOnA2) + .isNotEqualTo(metadataOnA1) + .isNotEqualTo(metadataOnB1) + .isNotEqualTo(metadataOnTest); + verifyRefState(catalogBranchA, tableIdentifier1, snapshotIdOnA2, 1); + + // repeat addRow() against branchB + catalogBranchB = initCatalog(branchB); + verifySchema(catalogBranchB, tableIdentifier1, Types.LongType.get(), Types.LongType.get()); + String metadataOnB2 = addRow(catalogBranchB, tableIdentifier1, "branch-b-2", + ImmutableMap.of("id0", 5L, "id2", 666L)); + long snapshotIdOnB2 = snapshotIdFromMetadata(catalogBranchA, metadataOnB2); + Assertions.assertThat(metadataOnB2) + .isNotEqualTo(metadataOnA1).isNotEqualTo(metadataOnA2) + .isNotEqualTo(metadataOnB1) + .isNotEqualTo(metadataOnTest); + verifyRefState(catalogBranchB, tableIdentifier1, snapshotIdOnB2, 2); + + // sanity check, branch "test" must not have changed + verifyRefState(catalog, tableIdentifier1, snapshotIdOnTest, 0); + } + + private void verifyRefState(NessieCatalog catalog, TableIdentifier identifier, long snapshotId, int schemaId) + throws Exception { + IcebergTable icebergTable = loadIcebergTable(catalog, identifier); + Assertions.assertThat(icebergTable) + .extracting(IcebergTable::getSnapshotId, IcebergTable::getSchemaId) + .containsExactly(snapshotId, schemaId); + } + + private long snapshotIdFromNessie(NessieCatalog catalog, TableIdentifier identifier) throws Exception { + IcebergTable icebergTable = loadIcebergTable(catalog, identifier); + return icebergTable.getSnapshotId(); + } + + private long snapshotIdFromMetadata(NessieCatalog catalog, String metadataLocation) { + Snapshot snapshot = TableMetadataParser.read(catalog.fileIO(), metadataLocation).currentSnapshot(); + return snapshot != null ? snapshot.snapshotId() : -1; + } + + private IcebergTable loadIcebergTable(NessieCatalog catalog, TableIdentifier identifier) + throws NessieNotFoundException { + ContentKey key = NessieUtil.toKey(identifier); + return api.getContent().refName(catalog.currentRefName()).key(key) + .get().get(key).unwrap(IcebergTable.class).get(); + } + + private String addRow(NessieCatalog catalog, TableIdentifier identifier, String fileName, Map data) + throws Exception { + Table table = catalog.loadTable(identifier); + GenericRecordBuilder recordBuilder = + new GenericRecordBuilder(AvroSchemaUtil.convert(table.schema(), table.name())); + data.forEach(recordBuilder::set); + + String fileLocation = writeRecordsToFile(table, table.schema(), fileName, + Collections.singletonList(recordBuilder.build())); + DataFile dataFile = makeDataFile(table, fileLocation); + + // Run via `Transaction` to exercise the whole code path ran via Spark (Spark SQL) + Transaction tx = table.newTransaction(); + tx.newAppend().appendFile(dataFile).commit(); + tx.commitTransaction(); + + return metadataLocation(catalog, identifier); + } + + private void verifySchema(NessieCatalog catalog, TableIdentifier identifier, Type... types) { + Assertions.assertThat(catalog.loadTable(identifier)) + .extracting(t -> t.schema().columns().stream().map(NestedField::type)) + .asInstanceOf(InstanceOfAssertFactories.stream(Type.class)) + .containsExactly(types); + } + private void updateSchema(NessieCatalog catalog, TableIdentifier identifier) { - catalog.loadTable(identifier).updateSchema().addColumn("id" + schemaCounter++, Types.LongType.get()).commit(); + updateSchema(catalog, identifier, Types.LongType.get()); + } + + private void updateSchema(NessieCatalog catalog, TableIdentifier identifier, Type type) { + // Run via `Transaction` to exercise the whole code path ran via Spark (Spark SQL) + Transaction tx = catalog.loadTable(identifier).newTransaction(); + tx.updateSchema().addColumn("id" + schemaCounter++, type).commit(); + tx.commitTransaction(); } private void testCatalogEquality( - NessieCatalog catalog, NessieCatalog compareCatalog, boolean table1Equal, boolean table2Equal) { + NessieCatalog catalog, NessieCatalog compareCatalog, boolean table1Equal, boolean table2Equal, + ThrowingCallable callable) { String testTable1 = metadataLocation(compareCatalog, tableIdentifier1); - String table1 = metadataLocation(catalog, tableIdentifier1); String testTable2 = metadataLocation(compareCatalog, tableIdentifier2); + + try { + callable.call(); + } catch (RuntimeException e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + + String table1 = metadataLocation(catalog, tableIdentifier1); String table2 = metadataLocation(catalog, tableIdentifier2); - Assertions.assertThat(table1.equals(testTable1)) - .withFailMessage(() -> String.format( - "Table %s on ref %s should%s equal table %s on ref %s", + AbstractStringAssert assertion = Assertions.assertThat(table1) + .describedAs("Table %s on ref %s should%s be equal to table %s on ref %s", tableIdentifier1.name(), - tableIdentifier2.name(), - table1Equal ? "" : " not", catalog.currentRefName(), - testCatalog.currentRefName())) - .isEqualTo(table1Equal); - - Assertions.assertThat(table2.equals(testTable2)) - .withFailMessage(() -> String.format( - "Table %s on ref %s should%s equal table %s on ref %s", + table1Equal ? "" : " not", tableIdentifier1.name(), + compareCatalog.currentRefName()); + if (table1Equal) { + assertion.isEqualTo(testTable1); + } else { + assertion.isNotEqualTo(testTable1); + } + + assertion = Assertions.assertThat(table2) + .describedAs("Table %s on ref %s should%s be equal to table %s on ref %s", tableIdentifier2.name(), - table1Equal ? "" : " not", catalog.currentRefName(), - testCatalog.currentRefName())) - .isEqualTo(table2Equal); + table2Equal ? "" : " not", + tableIdentifier2.name(), + compareCatalog.currentRefName()); + if (table2Equal) { + assertion.isEqualTo(testTable2); + } else { + assertion.isNotEqualTo(testTable2); + } } } diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java index 77a073c8f277..1641bc054db5 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java @@ -30,32 +30,28 @@ import org.apache.avro.generic.GenericRecordBuilder; import org.apache.hadoop.fs.Path; import org.apache.iceberg.DataFile; -import org.apache.iceberg.DataFiles; -import org.apache.iceberg.Files; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.avro.AvroSchemaUtil; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; -import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.types.Types; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.projectnessie.api.params.CommitLogParams; import org.projectnessie.error.NessieConflictException; import org.projectnessie.error.NessieNotFoundException; import org.projectnessie.model.Branch; import org.projectnessie.model.CommitMeta; -import org.projectnessie.model.ContentsKey; +import org.projectnessie.model.ContentKey; import org.projectnessie.model.IcebergTable; -import org.projectnessie.model.ImmutableOperations; -import org.projectnessie.model.ImmutablePut; +import org.projectnessie.model.LogResponse.LogEntry; +import org.projectnessie.model.Operation; import static org.apache.iceberg.TableMetadataParser.getFileExtension; import static org.apache.iceberg.types.Types.NestedField.optional; @@ -68,7 +64,7 @@ public class TestNessieTable extends BaseTestIceberg { private static final String DB_NAME = "db"; private static final String TABLE_NAME = "tbl"; private static final TableIdentifier TABLE_IDENTIFIER = TableIdentifier.of(DB_NAME, TABLE_NAME); - private static final ContentsKey KEY = ContentsKey.of(DB_NAME, TABLE_NAME); + private static final ContentKey KEY = ContentKey.of(DB_NAME, TABLE_NAME); private static final Schema schema = new Schema(Types.StructType.of( required(1, "id", Types.LongType.get())).fields()); private static final Schema altered = new Schema(Types.StructType.of( @@ -99,14 +95,92 @@ public void afterEach() throws Exception { super.afterEach(); } - private org.projectnessie.model.IcebergTable getTable(ContentsKey key) throws NessieNotFoundException { - return client.getContentsApi() - .getContents(key, BRANCH, null) - .unwrap(IcebergTable.class).get(); + private IcebergTable getTable(ContentKey key) + throws NessieNotFoundException { + return getTable(BRANCH, key); + } + + private IcebergTable getTable(String ref, ContentKey key) + throws NessieNotFoundException { + return api.getContent().key(key).refName(ref).get().get(key).unwrap(IcebergTable.class).get(); + } + + /** + * Verify that Nessie always returns the globally-current global-content w/ only DMLs. + */ + @Test + public void verifyGlobalStateMovesForDML() throws Exception { + // 1. initialize table + Table icebergTable = catalog.loadTable(TABLE_IDENTIFIER); + icebergTable.updateSchema().addColumn("initial_column", Types.LongType.get()).commit(); + + // 2. create 2nd branch + String testCaseBranch = "verify-global-moving"; + api.createReference().sourceRefName(BRANCH) + .reference(Branch.of(testCaseBranch, catalog.currentHash())).create(); + NessieCatalog branchCatalog = initCatalog(testCaseBranch); + + IcebergTable contentInitialMain = getTable(BRANCH, KEY); + IcebergTable contentInitialBranch = getTable(testCaseBranch, KEY); + Table tableInitialMain = catalog.loadTable(TABLE_IDENTIFIER); + + // verify table-metadata-location + snapshot-id + Assertions.assertThat(contentInitialMain) + .as("global-contents + snapshot-id equal on both branches in Nessie") + .isEqualTo(contentInitialBranch); + Assertions.assertThat(tableInitialMain.currentSnapshot()).isNull(); + + // 3. modify table in "main" branch (add some data) + + DataFile file1 = makeDataFile(icebergTable, addRecordsToFile(icebergTable, "file1")); + icebergTable.newAppend().appendFile(file1).commit(); + + IcebergTable contentsAfter1Main = getTable(KEY); + IcebergTable contentsAfter1Branch = getTable(testCaseBranch, KEY); + Table tableAfter1Main = catalog.loadTable(TABLE_IDENTIFIER); + + // --> assert getValue() against both branches returns the updated metadata-location + // verify table-metadata-location + Assertions.assertThat(contentInitialMain.getMetadataLocation()) + .describedAs("metadata-location must change on %s", BRANCH) + .isNotEqualTo(contentsAfter1Main.getMetadataLocation()); + Assertions.assertThat(contentInitialBranch.getMetadataLocation()) + .describedAs("metadata-location must change on %s", testCaseBranch) + .isNotEqualTo(contentsAfter1Branch.getMetadataLocation()); + Assertions.assertThat(contentsAfter1Main) + .extracting(IcebergTable::getSchemaId) + .describedAs("on-reference-state must not be equal on both branches") + .isEqualTo(contentsAfter1Branch.getSchemaId()); + // verify manifests + Assertions.assertThat(tableAfter1Main.currentSnapshot().allManifests()) + .describedAs("verify number of manifests on 'main'") + .hasSize(1); + + // 4. modify table in "main" branch (add some data) again + + DataFile file2 = makeDataFile(icebergTable, addRecordsToFile(icebergTable, "file2")); + icebergTable.newAppend().appendFile(file2).commit(); + + IcebergTable contentsAfter2Main = getTable(KEY); + IcebergTable contentsAfter2Branch = getTable(testCaseBranch, KEY); + Table tableAfter2Main = catalog.loadTable(TABLE_IDENTIFIER); + + // --> assert getValue() against both branches returns the updated metadata-location + // verify table-metadata-location + Assertions.assertThat(contentsAfter2Main.getMetadataLocation()) + .describedAs("metadata-location must change on %s", BRANCH) + .isNotEqualTo(contentsAfter1Main.getMetadataLocation()); + Assertions.assertThat(contentsAfter2Branch.getMetadataLocation()) + .describedAs("on-reference-state must change on %s", testCaseBranch) + .isNotEqualTo(contentsAfter1Branch.getMetadataLocation()); + // verify manifests + Assertions.assertThat(tableAfter2Main.currentSnapshot().allManifests()) + .describedAs("verify number of manifests on 'main'") + .hasSize(2); } @Test - public void testCreate() throws NessieNotFoundException, IOException { + public void testCreate() throws IOException { // Table should be created in iceberg // Table should be renamed in iceberg String tableName = TABLE_IDENTIFIER.name(); @@ -152,14 +226,16 @@ public void testRename() throws NessieNotFoundException { private void verifyCommitMetadata() throws NessieNotFoundException { // check that the author is properly set - List log = tree.getCommitLog(BRANCH, CommitLogParams.empty()).getOperations(); - Assertions.assertThat(log).isNotNull().isNotEmpty(); - log.forEach(commit -> { - Assertions.assertThat(commit.getAuthor()).isNotNull().isNotEmpty(); - Assertions.assertThat(commit.getAuthor()).isEqualTo(System.getProperty("user.name")); - Assertions.assertThat(commit.getProperties().get(NessieUtil.APPLICATION_TYPE)).isEqualTo("iceberg"); - Assertions.assertThat(commit.getMessage()).startsWith("iceberg"); - }); + List log = api.getCommitLog().refName(BRANCH).get().getLogEntries(); + Assertions.assertThat(log) + .isNotNull().isNotEmpty() + .allSatisfy(logEntry -> { + CommitMeta commit = logEntry.getCommitMeta(); + Assertions.assertThat(commit.getAuthor()).isNotNull().isNotEmpty(); + Assertions.assertThat(commit.getAuthor()).isEqualTo(System.getProperty("user.name")); + Assertions.assertThat(commit.getProperties().get(NessieUtil.APPLICATION_TYPE)).isEqualTo("iceberg"); + Assertions.assertThat(commit.getMessage()).startsWith("Iceberg"); + }); } @Test @@ -176,11 +252,7 @@ public void testDropWithoutPurgeLeavesTableData() throws IOException { String fileLocation = addRecordsToFile(table, "file"); - DataFile file = DataFiles.builder(table.spec()) - .withRecordCount(3) - .withPath(fileLocation) - .withFileSizeInBytes(Files.localInput(fileLocation).getLength()) - .build(); + DataFile file = makeDataFile(table, fileLocation); table.newAppend().appendFile(file).commit(); @@ -198,27 +270,11 @@ public void testDropWithoutPurgeLeavesTableData() throws IOException { public void testDropTable() throws IOException { Table table = catalog.loadTable(TABLE_IDENTIFIER); - GenericRecordBuilder recordBuilder = - new GenericRecordBuilder(AvroSchemaUtil.convert(schema, "test")); - List records = new ArrayList<>(); - records.add(recordBuilder.set("id", 1L).build()); - records.add(recordBuilder.set("id", 2L).build()); - records.add(recordBuilder.set("id", 3L).build()); - String location1 = addRecordsToFile(table, "file1"); String location2 = addRecordsToFile(table, "file2"); - DataFile file1 = DataFiles.builder(table.spec()) - .withRecordCount(3) - .withPath(location1) - .withFileSizeInBytes(Files.localInput(location2).getLength()) - .build(); - - DataFile file2 = DataFiles.builder(table.spec()) - .withRecordCount(3) - .withPath(location2) - .withFileSizeInBytes(Files.localInput(location1).getLength()) - .build(); + DataFile file1 = makeDataFile(table, location1); + DataFile file2 = makeDataFile(table, location2); // add both data files table.newAppend().appendFile(file1).appendFile(file2).commit(); @@ -240,11 +296,9 @@ public void testDropTable() throws IOException { for (ManifestFile manifest : manifests) { Assertions.assertThat(new File(manifest.path().replace("file:", ""))).exists(); } - Assertions.assertThat(new File( - ((HasTableOperations) table).operations() - .current() - .metadataFileLocation() - .replace("file:", ""))) + TableOperations ops = ((HasTableOperations) table).operations(); + String metadataLocation = ((NessieTableOperations) ops).currentMetadataLocation(); + Assertions.assertThat(new File(metadataLocation.replace("file:", ""))) .exists(); verifyCommitMetadata(); @@ -267,14 +321,15 @@ public void testExistingTableUpdate() { @Test public void testFailure() throws NessieNotFoundException, NessieConflictException { Table icebergTable = catalog.loadTable(TABLE_IDENTIFIER); - Branch branch = (Branch) client.getTreeApi().getReferenceByName(BRANCH); + Branch branch = (Branch) api.getReference().refName(BRANCH).get(); - IcebergTable table = client.getContentsApi().getContents(KEY, BRANCH, null).unwrap(IcebergTable.class).get(); + IcebergTable table = getTable(BRANCH, KEY); - client.getTreeApi().commitMultipleOperations(branch.getName(), branch.getHash(), - ImmutableOperations.builder().addOperations( - ImmutablePut.builder().key(KEY).contents(IcebergTable.of("dummytable.metadata.json")) - .build()).commitMeta(CommitMeta.fromMessage("")).build()); + IcebergTable value = IcebergTable.of("dummytable.metadata.json", 42, 42, 42, 42, "cid"); + api.commitMultipleOperations().branch(branch) + .operation(Operation.Put.of(KEY, value)) + .commitMeta(CommitMeta.fromMessage("")) + .commit(); Assertions.assertThatThrownBy(() -> icebergTable.updateSchema().addColumn("data", Types.LongType.get()).commit()) .isInstanceOf(CommitFailedException.class) @@ -342,16 +397,7 @@ private static String addRecordsToFile(Table table, String filename) throws IOEx records.add(recordBuilder.set("id", 2L).build()); records.add(recordBuilder.set("id", 3L).build()); - String fileLocation = table.location().replace("file:", "") + - String.format("/data/%s.avro", filename); - try (FileAppender writer = Avro.write(Files.localOutput(fileLocation)) - .schema(schema) - .named("test") - .build()) { - for (GenericData.Record rec : records) { - writer.add(rec); - } - } - return fileLocation; + return writeRecordsToFile(table, schema, filename, records); } + } diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestTableReference.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestTableReference.java deleted file mode 100644 index bd693810b30c..000000000000 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestTableReference.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.nessie; - -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -public class TestTableReference { - - @Test - public void noMarkings() { - String path = "foo"; - TableReference pti = TableReference.parse(path); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(pti.reference()).isNull(); - Assertions.assertThat(pti.timestamp()).isNull(); - } - - @Test - public void branchOnly() { - String path = "foo@bar"; - TableReference pti = TableReference.parse(path); - Assertions.assertThat("foo").isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat("bar").isEqualTo(pti.reference()); - Assertions.assertThat(pti.timestamp()).isNull(); - } - - @Test - public void timestampOnly() { - String path = "foo#baz"; - Assertions.assertThatThrownBy(() -> TableReference.parse(path)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid table name: # is not allowed (reference by timestamp is not supported)"); - } - - @Test - public void branchAndTimestamp() { - String path = "foo@bar#baz"; - Assertions.assertThatThrownBy(() -> TableReference.parse(path)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid table name: # is not allowed (reference by timestamp is not supported)"); - } - - @Test - public void twoBranches() { - String path = "foo@bar@boo"; - Assertions.assertThatThrownBy(() -> TableReference.parse(path)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Can only reference one branch in"); - } - - @Test - public void twoTimestamps() { - String path = "foo#baz#baa"; - Assertions.assertThatThrownBy(() -> TableReference.parse(path)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Can only reference one timestamp in"); - } - - @Test - public void strangeCharacters() { - String branch = "bar"; - String path = "/%"; - TableReference pti = TableReference.parse(path); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(pti.reference()).isNull(); - Assertions.assertThat(pti.timestamp()).isNull(); - pti = TableReference.parse(path + "@" + branch); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(branch).isEqualTo(pti.reference()); - Assertions.assertThat(pti.timestamp()).isNull(); - path = "&&"; - pti = TableReference.parse(path); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(pti.reference()).isNull(); - Assertions.assertThat(pti.timestamp()).isNull(); - pti = TableReference.parse(path + "@" + branch); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(branch).isEqualTo(pti.reference()); - Assertions.assertThat(pti.timestamp()).isNull(); - } - - @Test - public void doubleByte() { - String branch = "bar"; - String path = "/%国"; - TableReference pti = TableReference.parse(path); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(pti.reference()).isNull(); - Assertions.assertThat(pti.timestamp()).isNull(); - pti = TableReference.parse(path + "@" + branch); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(branch).isEqualTo(pti.reference()); - Assertions.assertThat(pti.timestamp()).isNull(); - path = "国.国"; - pti = TableReference.parse(path); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().toString()); - Assertions.assertThat(pti.reference()).isNull(); - Assertions.assertThat(pti.timestamp()).isNull(); - pti = TableReference.parse(path + "@" + branch); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().toString()); - Assertions.assertThat(branch).isEqualTo(pti.reference()); - Assertions.assertThat(pti.timestamp()).isNull(); - } - - @Test - public void whitespace() { - String branch = "bar "; - String path = "foo "; - TableReference pti = TableReference.parse(path); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(pti.reference()).isNull(); - Assertions.assertThat(pti.timestamp()).isNull(); - pti = TableReference.parse(path + "@" + branch); - Assertions.assertThat(path).isEqualTo(pti.tableIdentifier().name()); - Assertions.assertThat(branch).isEqualTo(pti.reference()); - Assertions.assertThat(pti.timestamp()).isNull(); - } -} diff --git a/site/mkdocs.yml b/site/mkdocs.yml index e09b3b50ea25..a6c232f79340 100644 --- a/site/mkdocs.yml +++ b/site/mkdocs.yml @@ -43,7 +43,7 @@ extra: Spark: /getting-started/ versions: iceberg: 0.12.1 - nessie: 0.9.2 + nessie: 0.14.0 logo: img/favicon-96x96.png version: 0.12.1 article_nav_top: false diff --git a/versions.props b/versions.props index abe0f80c4abb..bf19bc9ee8c6 100644 --- a/versions.props +++ b/versions.props @@ -24,7 +24,7 @@ javax.activation:activation = 1.1.1 org.glassfish.jaxb:jaxb-runtime = 2.3.3 software.amazon.awssdk:* = 2.15.7 org.scala-lang:scala-library = 2.12.10 -org.projectnessie:* = 0.9.2 +org.projectnessie:* = 0.15.1 # test deps org.junit.vintage:junit-vintage-engine = 5.7.2 From 169ff4798d596e6e0725d63059cf21762eb6e161 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 2 Dec 2021 15:03:16 +0100 Subject: [PATCH 2/2] Update site/mkdocs.yml Co-authored-by: Ajantha Bhat --- site/mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/mkdocs.yml b/site/mkdocs.yml index a6c232f79340..a287473eab15 100644 --- a/site/mkdocs.yml +++ b/site/mkdocs.yml @@ -43,7 +43,7 @@ extra: Spark: /getting-started/ versions: iceberg: 0.12.1 - nessie: 0.14.0 + nessie: 0.15.1 logo: img/favicon-96x96.png version: 0.12.1 article_nav_top: false