Bump Nessie to 0.54.0

nessie/src/main/java/org/apache/iceberg/nessie/NessieIcebergClient.java

@@ -206,6 +206,7 @@ public List<Namespace> listNamespaces(Namespace namespace) throws NoSuchNamespac
               .get();
       return response.getNamespaces().stream()
           .map(ns -> Namespace.of(ns.getElements().toArray(new String[0])))
+          .filter(ns -> ns.length() == namespace.length() + 1)
           .collect(Collectors.toList());
     } catch (NessieReferenceNotFoundException e) {
       throw new RuntimeException(

nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java

@@ -35,7 +35,9 @@
 import org.apache.iceberg.Table;
 import org.apache.iceberg.TableOperations;
 import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.catalog.Namespace;
 import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
 import org.apache.iceberg.io.FileAppender;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
@@ -153,6 +155,7 @@ NessieCatalog initCatalog(String ref, String hash) {
 
   protected Table createTable(TableIdentifier tableIdentifier, int count) {
     try {
+      createMissingNamespaces(tableIdentifier);
       return catalog.createTable(tableIdentifier, schema(count));
     } catch (Throwable t) {
       LOG.error("unable to do create " + tableIdentifier.toString(), t);
@@ -161,10 +164,37 @@ protected Table createTable(TableIdentifier tableIdentifier, int count) {
   }
 
   protected void createTable(TableIdentifier tableIdentifier) {
+    createMissingNamespaces(tableIdentifier);
     Schema schema = new Schema(StructType.of(required(1, "id", LongType.get())).fields());
     catalog.createTable(tableIdentifier, schema).location();
   }
 
+  protected Table createTable(TableIdentifier tableIdentifier, Schema schema) {
+    createMissingNamespaces(tableIdentifier);
+    return catalog.createTable(tableIdentifier, schema);
+  }
+
+  protected void createMissingNamespaces(TableIdentifier tableIdentifier) {
+    createMissingNamespaces(catalog, tableIdentifier);
+  }
+
+  protected static void createMissingNamespaces(
+      NessieCatalog catalog, TableIdentifier tableIdentifier) {
+    createMissingNamespaces(catalog, tableIdentifier.namespace());
+  }
+
+  protected static void createMissingNamespaces(NessieCatalog catalog, Namespace namespace) {
+    List<String> elements = Lists.newArrayList();
+    for (int i = 0; i < namespace.length(); i++) {
+      elements.add(namespace.level(i));
+      try {
+        catalog.createNamespace(Namespace.of(elements.toArray(new String[0])));
+      } catch (AlreadyExistsException ignore) {
+        // ignore
+      }
+    }
+  }
+
   protected static Schema schema(int count) {
     List<Types.NestedField> fields = Lists.newArrayList();
     for (int i = 0; i < count; i++) {

nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java

@@ -20,6 +20,7 @@
 
 import static org.apache.iceberg.types.Types.NestedField.required;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Map;
 import org.apache.avro.generic.GenericRecordBuilder;
@@ -452,37 +453,42 @@ public void testWithRefAndHash() throws NessieConflictException, NessieNotFoundE
 
     NessieCatalog nessieCatalog = initCatalog(testBranch);
     String hashBeforeNamespaceCreation = api.getReference().refName(testBranch).get().getHash();
-    Namespace namespace = Namespace.of("a", "b");
-    Assertions.assertThat(nessieCatalog.listNamespaces(namespace)).isEmpty();
+    Namespace namespaceA = Namespace.of("a");
+    Namespace namespaceAB = Namespace.of("a", "b");
+    Assertions.assertThat(nessieCatalog.listNamespaces(namespaceAB)).isEmpty();
 
-    nessieCatalog.createNamespace(namespace);
-    Assertions.assertThat(nessieCatalog.listNamespaces(namespace)).isNotEmpty();
-    Assertions.assertThat(nessieCatalog.listTables(namespace)).isEmpty();
+    createMissingNamespaces(
+        nessieCatalog, Namespace.of(Arrays.copyOf(namespaceAB.levels(), namespaceAB.length() - 1)));
+    nessieCatalog.createNamespace(namespaceAB);
+    Assertions.assertThat(nessieCatalog.listNamespaces(namespaceAB)).isEmpty();
+    Assertions.assertThat(nessieCatalog.listNamespaces(namespaceA)).containsExactly(namespaceAB);
+    Assertions.assertThat(nessieCatalog.listTables(namespaceAB)).isEmpty();
 
     NessieCatalog catalogAtHash1 = initCatalog(testBranch, hashBeforeNamespaceCreation);
-    Assertions.assertThat(catalogAtHash1.listNamespaces(namespace)).isEmpty();
-    Assertions.assertThat(catalogAtHash1.listTables(namespace)).isEmpty();
+    Assertions.assertThat(catalogAtHash1.listNamespaces(namespaceAB)).isEmpty();
+    Assertions.assertThat(catalogAtHash1.listTables(namespaceAB)).isEmpty();
 
-    TableIdentifier identifier = TableIdentifier.of(namespace, "table");
+    TableIdentifier identifier = TableIdentifier.of(namespaceAB, "table");
     String hashBeforeTableCreation = nessieCatalog.currentHash();
     nessieCatalog.createTable(identifier, schema);
-    Assertions.assertThat(nessieCatalog.listTables(namespace)).hasSize(1);
+    Assertions.assertThat(nessieCatalog.listTables(namespaceAB)).hasSize(1);
 
     NessieCatalog catalogAtHash2 = initCatalog(testBranch, hashBeforeTableCreation);
-    Assertions.assertThat(catalogAtHash2.listNamespaces(namespace)).isNotEmpty();
-    Assertions.assertThat(catalogAtHash2.listTables(namespace)).isEmpty();
+    Assertions.assertThat(catalogAtHash2.listNamespaces(namespaceAB)).isEmpty();
+    Assertions.assertThat(catalogAtHash2.listNamespaces(namespaceA)).containsExactly(namespaceAB);
+    Assertions.assertThat(catalogAtHash2.listTables(namespaceAB)).isEmpty();
 
     // updates should not be possible
     Assertions.assertThatThrownBy(() -> catalogAtHash2.createTable(identifier, schema))
         .isInstanceOf(IllegalArgumentException.class)
         .hasMessage("You can only mutate tables when using a branch without a hash or timestamp.");
-    Assertions.assertThat(catalogAtHash2.listTables(namespace)).isEmpty();
+    Assertions.assertThat(catalogAtHash2.listTables(namespaceAB)).isEmpty();
 
     // updates should be still possible here
     nessieCatalog = initCatalog(testBranch);
-    TableIdentifier identifier2 = TableIdentifier.of(namespace, "table2");
+    TableIdentifier identifier2 = TableIdentifier.of(namespaceAB, "table2");
     nessieCatalog.createTable(identifier2, schema);
-    Assertions.assertThat(nessieCatalog.listTables(namespace)).hasSize(2);
+    Assertions.assertThat(nessieCatalog.listTables(namespaceAB)).hasSize(2);
   }
 
   @Test
@@ -503,10 +509,13 @@ public void testDifferentTableSameName() throws NessieConflictException, NessieN
     TableIdentifier identifier = TableIdentifier.of("db", "table1");
 
     NessieCatalog nessieCatalog = initCatalog(branch1);
+
+    createMissingNamespaces(nessieCatalog, identifier);
     Table table1 = nessieCatalog.createTable(identifier, schema1);
     Assertions.assertThat(table1.schema().asStruct()).isEqualTo(schema1.asStruct());
 
     nessieCatalog = initCatalog(branch2);
+    createMissingNamespaces(nessieCatalog, identifier);
     Table table2 = nessieCatalog.createTable(identifier, schema2);
     Assertions.assertThat(table2.schema().asStruct()).isEqualTo(schema2.asStruct());
 

nessie/src/test/java/org/apache/iceberg/nessie/TestNamespace.java

@@ -47,30 +47,36 @@ public TestNamespace() {
 
   @Test
   public void testListNamespaces() {
-    createTable(TableIdentifier.parse("a.b.c.t1"));
-    createTable(TableIdentifier.parse("a.b.t2"));
-    createTable(TableIdentifier.parse("a.t3"));
-    createTable(TableIdentifier.parse("b.c.t4"));
-    createTable(TableIdentifier.parse("b.t5"));
-    createTable(TableIdentifier.parse("t6"));
-
-    List<TableIdentifier> tables = catalog.listTables(Namespace.of("a", "b", "c"));
+    Namespace nsA = Namespace.of("a");
+    Namespace nsAB = Namespace.of("a", "b");
+    Namespace nsABC = Namespace.of("a", "b", "c");
+    Namespace nsB = Namespace.of("b");
+    Namespace nsBC = Namespace.of("b", "c");
+
+    createTable(TableIdentifier.of(nsABC, "t1"));
+    createTable(TableIdentifier.of(nsAB, "t2"));
+    createTable(TableIdentifier.of(nsA, "t3"));
+    createTable(TableIdentifier.of(nsBC, "t4"));
+    createTable(TableIdentifier.of(nsB, "t5"));
+    createTable(TableIdentifier.of("t6"));
+
+    List<TableIdentifier> tables = catalog.listTables(nsABC);
     Assertions.assertThat(tables).isNotNull().hasSize(1);
-    tables = catalog.listTables(Namespace.of("a", "b"));
+    tables = catalog.listTables(nsAB);
     Assertions.assertThat(tables).isNotNull().hasSize(2);
-    tables = catalog.listTables(Namespace.of("a"));
+    tables = catalog.listTables(nsA);
     Assertions.assertThat(tables).isNotNull().hasSize(3);
     tables = catalog.listTables(null);
     Assertions.assertThat(tables).isNotNull().hasSize(6);
 
     List<Namespace> namespaces = catalog.listNamespaces();
-    Assertions.assertThat(namespaces).isNotNull().hasSize(5);
-    namespaces = catalog.listNamespaces(Namespace.of("a"));
-    Assertions.assertThat(namespaces).isNotNull().hasSize(3);
-    namespaces = catalog.listNamespaces(Namespace.of("a", "b"));
-    Assertions.assertThat(namespaces).isNotNull().hasSize(2);
-    namespaces = catalog.listNamespaces(Namespace.of("b"));
-    Assertions.assertThat(namespaces).isNotNull().hasSize(2);
+    Assertions.assertThat(namespaces).containsExactly(nsA, nsB);
+    namespaces = catalog.listNamespaces(nsA);
+    Assertions.assertThat(namespaces).containsExactly(nsAB);
+    namespaces = catalog.listNamespaces(nsAB);
+    Assertions.assertThat(namespaces).containsExactly(nsABC);
+    namespaces = catalog.listNamespaces(nsB);
+    Assertions.assertThat(namespaces).containsExactly(nsBC);
   }
 
   @Test

nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java

@@ -134,6 +134,16 @@ protected NessieCatalog catalog() {
     return catalog;
   }
 
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNestedNamespaces() {
+    return true;
+  }
+
   @Override
   protected boolean supportsNamespaceProperties() {
     return true;

nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java

@@ -55,6 +55,7 @@
 import org.projectnessie.client.ext.NessieClientFactory;
 import org.projectnessie.client.ext.NessieClientUri;
 import org.projectnessie.error.NessieConflictException;
+import org.projectnessie.error.NessieNamespaceAlreadyExistsException;
 import org.projectnessie.error.NessieNotFoundException;
 import org.projectnessie.model.Branch;
 import org.projectnessie.model.CommitMeta;
@@ -93,8 +94,7 @@ public TestNessieTable() {
   public void beforeEach(NessieClientFactory clientFactory, @NessieClientUri URI nessieUri)
       throws IOException {
     super.beforeEach(clientFactory, nessieUri);
-    this.tableLocation =
-        catalog.createTable(TABLE_IDENTIFIER, schema).location().replaceFirst("file:", "");
+    this.tableLocation = createTable(TABLE_IDENTIFIER, schema).location().replaceFirst("file:", "");
   }
 
   @Override
@@ -326,6 +326,7 @@ private void verifyCommitMetadata() throws NessieNotFoundException {
     Assertions.assertThat(log)
         .isNotNull()
         .isNotEmpty()
+        .filteredOn(e -> !e.getCommitMeta().getMessage().startsWith("create namespace "))
         .allSatisfy(
             logEntry -> {
               CommitMeta commit = logEntry.getCommitMeta();
@@ -429,12 +430,17 @@ private void validateRegister(TableIdentifier identifier, String metadataVersion
   }
 
   @Test
-  public void testRegisterTableWithGivenBranch() {
+  public void testRegisterTableWithGivenBranch() throws Exception {
     List<String> metadataVersionFiles = metadataVersionFiles(tableLocation);
     Assertions.assertThat(1).isEqualTo(metadataVersionFiles.size());
     ImmutableTableReference tableReference =
         ImmutableTableReference.builder().reference("main").name(TABLE_NAME).build();
     TableIdentifier identifier = TableIdentifier.of(DB_NAME, tableReference.toString());
+    try {
+      api.createNamespace().namespace(DB_NAME).refName(tableReference.getReference()).create();
+    } catch (NessieNamespaceAlreadyExistsException ignore) {
+      // ignore
+    }
     validateRegister(identifier, metadataVersionFiles.get(0));
   }
 
@@ -494,12 +500,17 @@ public void testRegisterTableWithDefaultBranch() {
   }
 
   @Test
-  public void testRegisterTableMoreThanOneBranch() {
+  public void testRegisterTableMoreThanOneBranch() throws Exception {
     List<String> metadataVersionFiles = metadataVersionFiles(tableLocation);
     Assertions.assertThat(1).isEqualTo(metadataVersionFiles.size());
     ImmutableTableReference tableReference =
         ImmutableTableReference.builder().reference("main").name(TABLE_NAME).build();
     TableIdentifier identifier = TableIdentifier.of(DB_NAME, tableReference.toString());
+    try {
+      api.createNamespace().namespace(DB_NAME).refName(tableReference.getReference()).create();
+    } catch (NessieNamespaceAlreadyExistsException ignore) {
+      // ignore
+    }
     validateRegister(identifier, metadataVersionFiles.get(0));
     Assertions.assertThat(catalog.dropTable(TABLE_IDENTIFIER, false)).isTrue();
     validateRegister(TABLE_IDENTIFIER, metadataVersionFiles.get(0));

versions.props

@@ -22,7 +22,7 @@ javax.xml.bind:jaxb-api = 2.3.1
 javax.activation:activation = 1.1.1
 org.glassfish.jaxb:jaxb-runtime = 2.3.3
 software.amazon.awssdk:* = 2.20.18
-org.projectnessie.nessie:* = 0.51.1
+org.projectnessie.nessie:* = 0.54.0
 com.google.cloud:libraries-bom = 24.1.0
 org.scala-lang.modules:scala-collection-compat_2.12 = 2.6.0
 org.scala-lang.modules:scala-collection-compat_2.13 = 2.6.0