diff --git a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java index c0f6515ca7fc..dfeea821936a 100644 --- a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java +++ b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java @@ -499,6 +499,10 @@ public void testBasicCreateTable() { Assert.assertFalse("Table should not exist", catalog.tableExists(ident)); + if (requiresNamespaceCreate()) { + catalog.createNamespace(ident.namespace()); + } + Table table = catalog.buildTable(ident, SCHEMA).create(); Assert.assertTrue("Table should exist", catalog.tableExists(ident)); @@ -573,6 +577,10 @@ public void testBasicCreateTableThatAlreadyExists() { TableIdentifier ident = TableIdentifier.of("ns", "table"); + if (requiresNamespaceCreate()) { + catalog.createNamespace(ident.namespace()); + } + Assert.assertFalse("Table should not exist", catalog.tableExists(ident)); catalog.buildTable(ident, SCHEMA).create(); @@ -597,6 +605,10 @@ public void testCompleteCreateTable() { TableIdentifier ident = TableIdentifier.of("ns", "table"); + if (requiresNamespaceCreate()) { + catalog.createNamespace(ident.namespace()); + } + Assert.assertFalse("Table should not exist", catalog.tableExists(ident)); Map properties = @@ -633,6 +645,10 @@ public void testLoadTable() { TableIdentifier ident = TableIdentifier.of("ns", "table"); + if (requiresNamespaceCreate()) { + catalog.createNamespace(ident.namespace()); + } + Assert.assertFalse("Table should not exist", catalog.tableExists(ident)); Map properties = @@ -671,6 +687,10 @@ public void testLoadMetadataTable() { TableIdentifier tableIdent = TableIdentifier.of("ns", "table"); TableIdentifier metaIdent = TableIdentifier.of("ns", "table", "files"); + if (requiresNamespaceCreate()) { + catalog.createNamespace(tableIdent.namespace()); + } + catalog.buildTable(tableIdent, SCHEMA).create(); Table table = catalog.loadTable(metaIdent); @@ -934,6 +954,10 @@ public void testListTables() { public void testUpdateTableSchema() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); UpdateSchema update = table.updateSchema().addColumn("new_col", Types.LongType.get()); @@ -953,6 +977,10 @@ public void testUpdateTableSchema() { public void testUUIDValidation() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); UpdateSchema update = table.updateSchema().addColumn("new_col", Types.LongType.get()); @@ -981,6 +1009,10 @@ public void testUpdateTableSchemaServerSideRetry() { supportsServerSideRetry()); C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); UpdateSchema update = table.updateSchema().addColumn("new_col", Types.LongType.get()); @@ -1003,6 +1035,10 @@ public void testUpdateTableSchemaServerSideRetry() { public void testUpdateTableSchemaConflict() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); UpdateSchema update = table.updateSchema().addColumn("new_col", Types.LongType.get()); @@ -1032,6 +1068,10 @@ public void testUpdateTableSchemaConflict() { public void testUpdateTableSchemaAssignmentConflict() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); UpdateSchema update = table.updateSchema().addColumn("new_col", Types.LongType.get()); @@ -1064,6 +1104,10 @@ public void testUpdateTableSchemaAssignmentConflict() { public void testUpdateTableSchemaThenRevert() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); table @@ -1085,6 +1129,10 @@ public void testUpdateTableSchemaThenRevert() { public void testUpdateTableSpec() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); UpdatePartitionSpec update = table.updateSpec().addField("shard", Expressions.bucket("id", 16)); @@ -1105,6 +1153,10 @@ public void testUpdateTableSpecServerSideRetry() { "Spec update recovery is only supported with server-side retry", supportsServerSideRetry()); C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); UpdatePartitionSpec update = table.updateSpec().addField("shard", Expressions.bucket("id", 16)); @@ -1131,6 +1183,10 @@ public void testUpdateTableSpecServerSideRetry() { public void testUpdateTableSpecConflict() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).withPartitionSpec(SPEC).create(); UpdatePartitionSpec update = @@ -1164,6 +1220,10 @@ public void testUpdateTableSpecConflict() { public void testUpdateTableAssignmentSpecConflict() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); UpdatePartitionSpec update = table.updateSpec().addField("shard", Expressions.bucket("id", 16)); @@ -1196,6 +1256,10 @@ public void testUpdateTableAssignmentSpecConflict() { public void testUpdateTableSpecThenRevert() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + // create a v2 table. otherwise the spec update would produce a different spec with a void // partition field Table table = @@ -1218,6 +1282,10 @@ public void testUpdateTableSpecThenRevert() { public void testUpdateTableSortOrder() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); ReplaceSortOrder update = table.replaceSortOrder().asc(Expressions.bucket("id", 16)).asc("id"); @@ -1239,6 +1307,10 @@ public void testUpdateTableSortOrderServerSideRetry() { supportsServerSideRetry()); C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); ReplaceSortOrder update = table.replaceSortOrder().asc(Expressions.bucket("id", 16)).asc("id"); @@ -1265,6 +1337,10 @@ public void testUpdateTableSortOrderServerSideRetry() { public void testUpdateTableOrderThenRevert() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).withSortOrder(WRITE_ORDER).create(); table.replaceSortOrder().asc("id").commit(); @@ -1279,6 +1355,10 @@ public void testUpdateTableOrderThenRevert() { public void testAppend() throws IOException { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).withPartitionSpec(SPEC).create(); try (CloseableIterable tasks = table.newScan().planFiles()) { @@ -1294,6 +1374,10 @@ public void testAppend() throws IOException { public void testConcurrentAppendEmptyTable() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).withPartitionSpec(SPEC).create(); assertNoFiles(table); @@ -1314,6 +1398,10 @@ public void testConcurrentAppendEmptyTable() { public void testConcurrentAppendNonEmptyTable() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).withPartitionSpec(SPEC).create(); assertNoFiles(table); @@ -1339,6 +1427,10 @@ public void testConcurrentAppendNonEmptyTable() { public void testUpdateTransaction() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); Transaction transaction = table.newTransaction(); @@ -1371,6 +1463,10 @@ public void testUpdateTransaction() { public void testCreateTransaction() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction create = catalog.buildTable(TABLE, SCHEMA).createTransaction(); Assert.assertFalse( @@ -1392,6 +1488,10 @@ public void testCreateTransaction() { public void testCompleteCreateTransaction() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Map properties = ImmutableMap.of("user", "someone", "created-at", "2022-02-25T00:38:19"); Transaction create = @@ -1440,6 +1540,10 @@ public void testCompleteCreateTransaction() { public void testCompleteCreateTransactionMultipleSchemas() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Map properties = ImmutableMap.of("user", "someone", "created-at", "2022-02-25T00:38:19"); Transaction create = @@ -1525,6 +1629,10 @@ public void testCompleteCreateTransactionMultipleSchemas() { public void testCompleteCreateTransactionV2() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Map properties = ImmutableMap.of( "user", "someone", "created-at", "2022-02-25T00:38:19", "format-version", "2"); @@ -1582,6 +1690,10 @@ public void testCompleteCreateTransactionV2() { public void testConcurrentCreateTransaction() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction create = catalog.buildTable(TABLE, SCHEMA).createTransaction(); Assert.assertFalse( @@ -1617,6 +1729,10 @@ public void testConcurrentCreateTransaction() { public void testCreateOrReplaceTransactionCreate() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction create = catalog.buildTable(TABLE, SCHEMA).createOrReplaceTransaction(); Assert.assertFalse( @@ -1638,6 +1754,10 @@ public void testCreateOrReplaceTransactionCreate() { public void testCompleteCreateOrReplaceTransactionCreate() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Map properties = ImmutableMap.of("user", "someone", "created-at", "2022-02-25T00:38:19"); Transaction createOrReplace = @@ -1686,6 +1806,10 @@ public void testCompleteCreateOrReplaceTransactionCreate() { public void testCreateOrReplaceReplaceTransactionReplace() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table original = catalog.buildTable(TABLE, OTHER_SCHEMA).create(); Assert.assertTrue("Table should exist before replaceTransaction", catalog.tableExists(TABLE)); @@ -1727,6 +1851,10 @@ public void testCreateOrReplaceReplaceTransactionReplace() { public void testCompleteCreateOrReplaceTransactionReplace() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table original = catalog.buildTable(TABLE, OTHER_SCHEMA).create(); Assert.assertTrue("Table should exist before replaceTransaction", catalog.tableExists(TABLE)); @@ -1796,6 +1924,10 @@ public void testCreateOrReplaceTransactionConcurrentCreate() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction createOrReplace = catalog.buildTable(TABLE, SCHEMA).createOrReplaceTransaction(); Assert.assertFalse( @@ -1830,6 +1962,10 @@ public void testCreateOrReplaceTransactionConcurrentCreate() { public void testReplaceTransaction() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table original = catalog.buildTable(TABLE, OTHER_SCHEMA).create(); Assert.assertTrue("Table should exist before replaceTransaction", catalog.tableExists(TABLE)); @@ -1871,6 +2007,10 @@ public void testReplaceTransaction() { public void testCompleteReplaceTransaction() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table original = catalog.buildTable(TABLE, OTHER_SCHEMA).create(); Assert.assertTrue("Table should exist before replaceTransaction", catalog.tableExists(TABLE)); @@ -1936,6 +2076,10 @@ public void testCompleteReplaceTransaction() { public void testReplaceTransactionRequiresTableExists() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + AssertHelpers.assertThrows( "Should fail to create replace transaction with a missing table", NoSuchTableException.class, @@ -1947,6 +2091,10 @@ public void testReplaceTransactionRequiresTableExists() { public void testConcurrentReplaceTransactions() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -1988,6 +2136,10 @@ public void testConcurrentReplaceTransactions() { public void testConcurrentReplaceTransactionSchema() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, OTHER_SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -2025,6 +2177,10 @@ public void testConcurrentReplaceTransactionSchema() { public void testConcurrentReplaceTransactionSchema2() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, OTHER_SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -2064,6 +2220,10 @@ public void testConcurrentReplaceTransactionSchemaConflict() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, OTHER_SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -2099,6 +2259,10 @@ public void testConcurrentReplaceTransactionSchemaConflict() { public void testConcurrentReplaceTransactionPartitionSpec() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -2134,6 +2298,10 @@ public void testConcurrentReplaceTransactionPartitionSpec() { public void testConcurrentReplaceTransactionPartitionSpec2() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -2170,6 +2338,10 @@ public void testConcurrentReplaceTransactionPartitionSpecConflict() { Assume.assumeTrue("Spec conflicts are detected server-side", supportsServerSideRetry()); C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -2207,6 +2379,10 @@ public void testConcurrentReplaceTransactionPartitionSpecConflict() { public void testConcurrentReplaceTransactionSortOrder() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -2240,6 +2416,10 @@ public void testConcurrentReplaceTransactionSortOrder() { public void testConcurrentReplaceTransactionSortOrderConflict() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Transaction transaction = catalog.buildTable(TABLE, SCHEMA).createTransaction(); transaction.newFastAppend().appendFile(FILE_A).commit(); transaction.commitTransaction(); @@ -2280,6 +2460,10 @@ public void testConcurrentReplaceTransactionSortOrderConflict() { public void testMetadataFileLocationsRemovalAfterCommit() { C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + Table table = catalog.buildTable(TABLE, SCHEMA).create(); table.updateSchema().addColumn("a", Types.LongType.get()).commit(); table.updateSchema().addColumn("b", Types.LongType.get()).commit(); diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieIcebergClient.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieIcebergClient.java index 596f72bb592d..5256b37d0c22 100644 --- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieIcebergClient.java +++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieIcebergClient.java @@ -206,6 +206,7 @@ public List listNamespaces(Namespace namespace) throws NoSuchNamespac .get(); return response.getNamespaces().stream() .map(ns -> Namespace.of(ns.getElements().toArray(new String[0]))) + .filter(ns -> ns.length() == namespace.length() + 1) .collect(Collectors.toList()); } catch (NessieReferenceNotFoundException e) { throw new RuntimeException( diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java index b66bbe1c42f0..ae4947a98cb7 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java @@ -35,7 +35,9 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableOperations; import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; @@ -153,6 +155,7 @@ NessieCatalog initCatalog(String ref, String hash) { protected Table createTable(TableIdentifier tableIdentifier, int count) { try { + createMissingNamespaces(tableIdentifier); return catalog.createTable(tableIdentifier, schema(count)); } catch (Throwable t) { LOG.error("unable to do create " + tableIdentifier.toString(), t); @@ -161,10 +164,37 @@ protected Table createTable(TableIdentifier tableIdentifier, int count) { } protected void createTable(TableIdentifier tableIdentifier) { + createMissingNamespaces(tableIdentifier); Schema schema = new Schema(StructType.of(required(1, "id", LongType.get())).fields()); catalog.createTable(tableIdentifier, schema).location(); } + protected Table createTable(TableIdentifier tableIdentifier, Schema schema) { + createMissingNamespaces(tableIdentifier); + return catalog.createTable(tableIdentifier, schema); + } + + protected void createMissingNamespaces(TableIdentifier tableIdentifier) { + createMissingNamespaces(catalog, tableIdentifier); + } + + protected static void createMissingNamespaces( + NessieCatalog catalog, TableIdentifier tableIdentifier) { + createMissingNamespaces(catalog, tableIdentifier.namespace()); + } + + protected static void createMissingNamespaces(NessieCatalog catalog, Namespace namespace) { + List elements = Lists.newArrayList(); + for (int i = 0; i < namespace.length(); i++) { + elements.add(namespace.level(i)); + try { + catalog.createNamespace(Namespace.of(elements.toArray(new String[0]))); + } catch (AlreadyExistsException ignore) { + // ignore + } + } + } + protected static Schema schema(int count) { List fields = Lists.newArrayList(); for (int i = 0; i < count; i++) { diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java index f2412ec64d5a..98e2834d6df4 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestBranchVisibility.java @@ -20,6 +20,7 @@ import static org.apache.iceberg.types.Types.NestedField.required; +import java.util.Arrays; import java.util.Collections; import java.util.Map; import org.apache.avro.generic.GenericRecordBuilder; @@ -452,37 +453,42 @@ public void testWithRefAndHash() throws NessieConflictException, NessieNotFoundE NessieCatalog nessieCatalog = initCatalog(testBranch); String hashBeforeNamespaceCreation = api.getReference().refName(testBranch).get().getHash(); - Namespace namespace = Namespace.of("a", "b"); - Assertions.assertThat(nessieCatalog.listNamespaces(namespace)).isEmpty(); + Namespace namespaceA = Namespace.of("a"); + Namespace namespaceAB = Namespace.of("a", "b"); + Assertions.assertThat(nessieCatalog.listNamespaces(namespaceAB)).isEmpty(); - nessieCatalog.createNamespace(namespace); - Assertions.assertThat(nessieCatalog.listNamespaces(namespace)).isNotEmpty(); - Assertions.assertThat(nessieCatalog.listTables(namespace)).isEmpty(); + createMissingNamespaces( + nessieCatalog, Namespace.of(Arrays.copyOf(namespaceAB.levels(), namespaceAB.length() - 1))); + nessieCatalog.createNamespace(namespaceAB); + Assertions.assertThat(nessieCatalog.listNamespaces(namespaceAB)).isEmpty(); + Assertions.assertThat(nessieCatalog.listNamespaces(namespaceA)).containsExactly(namespaceAB); + Assertions.assertThat(nessieCatalog.listTables(namespaceAB)).isEmpty(); NessieCatalog catalogAtHash1 = initCatalog(testBranch, hashBeforeNamespaceCreation); - Assertions.assertThat(catalogAtHash1.listNamespaces(namespace)).isEmpty(); - Assertions.assertThat(catalogAtHash1.listTables(namespace)).isEmpty(); + Assertions.assertThat(catalogAtHash1.listNamespaces(namespaceAB)).isEmpty(); + Assertions.assertThat(catalogAtHash1.listTables(namespaceAB)).isEmpty(); - TableIdentifier identifier = TableIdentifier.of(namespace, "table"); + TableIdentifier identifier = TableIdentifier.of(namespaceAB, "table"); String hashBeforeTableCreation = nessieCatalog.currentHash(); nessieCatalog.createTable(identifier, schema); - Assertions.assertThat(nessieCatalog.listTables(namespace)).hasSize(1); + Assertions.assertThat(nessieCatalog.listTables(namespaceAB)).hasSize(1); NessieCatalog catalogAtHash2 = initCatalog(testBranch, hashBeforeTableCreation); - Assertions.assertThat(catalogAtHash2.listNamespaces(namespace)).isNotEmpty(); - Assertions.assertThat(catalogAtHash2.listTables(namespace)).isEmpty(); + Assertions.assertThat(catalogAtHash2.listNamespaces(namespaceAB)).isEmpty(); + Assertions.assertThat(catalogAtHash2.listNamespaces(namespaceA)).containsExactly(namespaceAB); + Assertions.assertThat(catalogAtHash2.listTables(namespaceAB)).isEmpty(); // updates should not be possible Assertions.assertThatThrownBy(() -> catalogAtHash2.createTable(identifier, schema)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("You can only mutate tables when using a branch without a hash or timestamp."); - Assertions.assertThat(catalogAtHash2.listTables(namespace)).isEmpty(); + Assertions.assertThat(catalogAtHash2.listTables(namespaceAB)).isEmpty(); // updates should be still possible here nessieCatalog = initCatalog(testBranch); - TableIdentifier identifier2 = TableIdentifier.of(namespace, "table2"); + TableIdentifier identifier2 = TableIdentifier.of(namespaceAB, "table2"); nessieCatalog.createTable(identifier2, schema); - Assertions.assertThat(nessieCatalog.listTables(namespace)).hasSize(2); + Assertions.assertThat(nessieCatalog.listTables(namespaceAB)).hasSize(2); } @Test @@ -503,10 +509,13 @@ public void testDifferentTableSameName() throws NessieConflictException, NessieN TableIdentifier identifier = TableIdentifier.of("db", "table1"); NessieCatalog nessieCatalog = initCatalog(branch1); + + createMissingNamespaces(nessieCatalog, identifier); Table table1 = nessieCatalog.createTable(identifier, schema1); Assertions.assertThat(table1.schema().asStruct()).isEqualTo(schema1.asStruct()); nessieCatalog = initCatalog(branch2); + createMissingNamespaces(nessieCatalog, identifier); Table table2 = nessieCatalog.createTable(identifier, schema2); Assertions.assertThat(table2.schema().asStruct()).isEqualTo(schema2.asStruct()); diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNamespace.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNamespace.java index 3da8e72ad0d0..8267329678de 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNamespace.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNamespace.java @@ -47,30 +47,36 @@ public TestNamespace() { @Test public void testListNamespaces() { - createTable(TableIdentifier.parse("a.b.c.t1")); - createTable(TableIdentifier.parse("a.b.t2")); - createTable(TableIdentifier.parse("a.t3")); - createTable(TableIdentifier.parse("b.c.t4")); - createTable(TableIdentifier.parse("b.t5")); - createTable(TableIdentifier.parse("t6")); - - List tables = catalog.listTables(Namespace.of("a", "b", "c")); + Namespace nsA = Namespace.of("a"); + Namespace nsAB = Namespace.of("a", "b"); + Namespace nsABC = Namespace.of("a", "b", "c"); + Namespace nsB = Namespace.of("b"); + Namespace nsBC = Namespace.of("b", "c"); + + createTable(TableIdentifier.of(nsABC, "t1")); + createTable(TableIdentifier.of(nsAB, "t2")); + createTable(TableIdentifier.of(nsA, "t3")); + createTable(TableIdentifier.of(nsBC, "t4")); + createTable(TableIdentifier.of(nsB, "t5")); + createTable(TableIdentifier.of("t6")); + + List tables = catalog.listTables(nsABC); Assertions.assertThat(tables).isNotNull().hasSize(1); - tables = catalog.listTables(Namespace.of("a", "b")); + tables = catalog.listTables(nsAB); Assertions.assertThat(tables).isNotNull().hasSize(2); - tables = catalog.listTables(Namespace.of("a")); + tables = catalog.listTables(nsA); Assertions.assertThat(tables).isNotNull().hasSize(3); tables = catalog.listTables(null); Assertions.assertThat(tables).isNotNull().hasSize(6); List namespaces = catalog.listNamespaces(); - Assertions.assertThat(namespaces).isNotNull().hasSize(5); - namespaces = catalog.listNamespaces(Namespace.of("a")); - Assertions.assertThat(namespaces).isNotNull().hasSize(3); - namespaces = catalog.listNamespaces(Namespace.of("a", "b")); - Assertions.assertThat(namespaces).isNotNull().hasSize(2); - namespaces = catalog.listNamespaces(Namespace.of("b")); - Assertions.assertThat(namespaces).isNotNull().hasSize(2); + Assertions.assertThat(namespaces).containsExactly(nsA, nsB); + namespaces = catalog.listNamespaces(nsA); + Assertions.assertThat(namespaces).containsExactly(nsAB); + namespaces = catalog.listNamespaces(nsAB); + Assertions.assertThat(namespaces).containsExactly(nsABC); + namespaces = catalog.listNamespaces(nsB); + Assertions.assertThat(namespaces).containsExactly(nsBC); } @Test diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java index 080ad860bc87..f8ed3987c1ea 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java @@ -134,6 +134,16 @@ protected NessieCatalog catalog() { return catalog; } + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected boolean supportsNestedNamespaces() { + return true; + } + @Override protected boolean supportsNamespaceProperties() { return true; diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java index 0207ea810929..9881af5cfdde 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java @@ -55,6 +55,7 @@ import org.projectnessie.client.ext.NessieClientFactory; import org.projectnessie.client.ext.NessieClientUri; import org.projectnessie.error.NessieConflictException; +import org.projectnessie.error.NessieNamespaceAlreadyExistsException; import org.projectnessie.error.NessieNotFoundException; import org.projectnessie.model.Branch; import org.projectnessie.model.CommitMeta; @@ -93,8 +94,7 @@ public TestNessieTable() { public void beforeEach(NessieClientFactory clientFactory, @NessieClientUri URI nessieUri) throws IOException { super.beforeEach(clientFactory, nessieUri); - this.tableLocation = - catalog.createTable(TABLE_IDENTIFIER, schema).location().replaceFirst("file:", ""); + this.tableLocation = createTable(TABLE_IDENTIFIER, schema).location().replaceFirst("file:", ""); } @Override @@ -326,6 +326,7 @@ private void verifyCommitMetadata() throws NessieNotFoundException { Assertions.assertThat(log) .isNotNull() .isNotEmpty() + .filteredOn(e -> !e.getCommitMeta().getMessage().startsWith("create namespace ")) .allSatisfy( logEntry -> { CommitMeta commit = logEntry.getCommitMeta(); @@ -429,12 +430,17 @@ private void validateRegister(TableIdentifier identifier, String metadataVersion } @Test - public void testRegisterTableWithGivenBranch() { + public void testRegisterTableWithGivenBranch() throws Exception { List metadataVersionFiles = metadataVersionFiles(tableLocation); Assertions.assertThat(1).isEqualTo(metadataVersionFiles.size()); ImmutableTableReference tableReference = ImmutableTableReference.builder().reference("main").name(TABLE_NAME).build(); TableIdentifier identifier = TableIdentifier.of(DB_NAME, tableReference.toString()); + try { + api.createNamespace().namespace(DB_NAME).refName(tableReference.getReference()).create(); + } catch (NessieNamespaceAlreadyExistsException ignore) { + // ignore + } validateRegister(identifier, metadataVersionFiles.get(0)); } @@ -494,12 +500,17 @@ public void testRegisterTableWithDefaultBranch() { } @Test - public void testRegisterTableMoreThanOneBranch() { + public void testRegisterTableMoreThanOneBranch() throws Exception { List metadataVersionFiles = metadataVersionFiles(tableLocation); Assertions.assertThat(1).isEqualTo(metadataVersionFiles.size()); ImmutableTableReference tableReference = ImmutableTableReference.builder().reference("main").name(TABLE_NAME).build(); TableIdentifier identifier = TableIdentifier.of(DB_NAME, tableReference.toString()); + try { + api.createNamespace().namespace(DB_NAME).refName(tableReference.getReference()).create(); + } catch (NessieNamespaceAlreadyExistsException ignore) { + // ignore + } validateRegister(identifier, metadataVersionFiles.get(0)); Assertions.assertThat(catalog.dropTable(TABLE_IDENTIFIER, false)).isTrue(); validateRegister(TABLE_IDENTIFIER, metadataVersionFiles.get(0));