diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueCatalogNamespaceTest.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueCatalogNamespaceTest.java index d7f10585df07..570982f0074c 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueCatalogNamespaceTest.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueCatalogNamespaceTest.java @@ -27,6 +27,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; @@ -48,9 +49,19 @@ public void testCreateNamespace() { EntityNotFoundException.class, "not found", () -> glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build())); - glueCatalog.createNamespace(Namespace.of(namespace)); + Map properties = ImmutableMap.of( + IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description", + IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://location", + "key", "val"); + Namespace ns = Namespace.of(namespace); + glueCatalog.createNamespace(ns, properties); Database database = glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()).database(); Assert.assertEquals("namespace must equal database name", namespace, database.name()); + Assert.assertEquals("namespace description should be set", "description", database.description()); + Assert.assertEquals("namespace location should be set", "s3://location", database.locationUri()); + Assert.assertEquals("namespace parameters should be set", + ImmutableMap.of("key", "val"), database.parameters()); + Assert.assertEquals(properties, glueCatalog.loadNamespaceMetadata(ns)); } @Test @@ -100,27 +111,38 @@ public void testNamespaceProperties() { Map properties = Maps.newHashMap(); properties.put("key", "val"); properties.put("key2", "val2"); + properties.put(IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://test"); + properties.put(IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description"); glueCatalog.setProperties(Namespace.of(namespace), properties); Database database = glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()).database(); Assert.assertTrue(database.parameters().containsKey("key")); Assert.assertEquals("val", database.parameters().get("key")); Assert.assertTrue(database.parameters().containsKey("key2")); Assert.assertEquals("val2", database.parameters().get("key2")); + Assert.assertEquals("s3://test", database.locationUri()); + Assert.assertEquals("description", database.description()); // remove properties - glueCatalog.removeProperties(Namespace.of(namespace), Sets.newHashSet("key")); + glueCatalog.removeProperties(Namespace.of(namespace), Sets.newHashSet( + "key", IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY)); database = glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()).database(); Assert.assertFalse(database.parameters().containsKey("key")); Assert.assertTrue(database.parameters().containsKey("key2")); Assert.assertEquals("val2", database.parameters().get("key2")); + Assert.assertNull(database.locationUri()); + Assert.assertNull(database.description()); // add back properties = Maps.newHashMap(); properties.put("key", "val"); + properties.put(IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://test2"); + properties.put(IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description2"); glueCatalog.setProperties(Namespace.of(namespace), properties); database = glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()).database(); Assert.assertTrue(database.parameters().containsKey("key")); Assert.assertEquals("val", database.parameters().get("key")); Assert.assertTrue(database.parameters().containsKey("key2")); Assert.assertEquals("val2", database.parameters().get("key2")); + Assert.assertEquals("s3://test2", database.locationUri()); + Assert.assertEquals("description2", database.description()); } @Test diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java index fb4aac3f27e8..9729e87272ce 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java +++ b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java @@ -54,6 +54,7 @@ import software.amazon.awssdk.services.glue.GlueClient; import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest; import software.amazon.awssdk.services.glue.model.CreateTableRequest; +import software.amazon.awssdk.services.glue.model.Database; import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest; import software.amazon.awssdk.services.glue.model.DeleteTableRequest; import software.amazon.awssdk.services.glue.model.EntityNotFoundException; @@ -333,11 +334,21 @@ public List listNamespaces(Namespace namespace) throws NoSuchNamespac public Map loadNamespaceMetadata(Namespace namespace) throws NoSuchNamespaceException { String databaseName = IcebergToGlueConverter.toDatabaseName(namespace); try { - GetDatabaseResponse response = glue.getDatabase(GetDatabaseRequest.builder() + Database database = glue.getDatabase(GetDatabaseRequest.builder() .catalogId(awsProperties.glueCatalogId()) .name(databaseName) - .build()); - Map result = response.database().parameters(); + .build()) + .database(); + Map result = Maps.newHashMap(database.parameters()); + + if (database.locationUri() != null) { + result.put(IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, database.locationUri()); + } + + if (database.description() != null) { + result.put(IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, database.description()); + } + LOG.debug("Loaded metadata for namespace {} found {}", namespace, result); return result; } catch (InvalidInputException e) { diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java index e7725ad95ad6..6912ca78aee6 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java +++ b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java @@ -34,6 +34,7 @@ import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; @@ -55,6 +56,8 @@ private IcebergToGlueConverter() { private static final Pattern GLUE_DB_PATTERN = Pattern.compile("^[a-z0-9_]{1,252}$"); private static final Pattern GLUE_TABLE_PATTERN = Pattern.compile("^[a-z0-9_]{1,255}$"); + public static final String GLUE_DB_LOCATION_KEY = "location"; + public static final String GLUE_DB_DESCRIPTION_KEY = "comment"; public static final String ICEBERG_FIELD_USAGE = "iceberg.field.usage"; public static final String ICEBERG_FIELD_TYPE_TYPE_ID = "iceberg.field.type.typeid"; public static final String ICEBERG_FIELD_TYPE_STRING = "iceberg.field.type.string"; @@ -118,10 +121,19 @@ static String getDatabaseName(TableIdentifier tableIdentifier) { * @return Glue DatabaseInput */ static DatabaseInput toDatabaseInput(Namespace namespace, Map metadata) { - return DatabaseInput.builder() - .name(toDatabaseName(namespace)) - .parameters(metadata) - .build(); + DatabaseInput.Builder builder = DatabaseInput.builder().name(toDatabaseName(namespace)); + Map parameters = Maps.newHashMap(); + metadata.forEach((k, v) -> { + if (GLUE_DB_DESCRIPTION_KEY.equals(k)) { + builder.description(v); + } else if (GLUE_DB_LOCATION_KEY.equals(k)) { + builder.locationUri(v); + } else { + parameters.put(k, v); + } + }); + + return builder.parameters(parameters).build(); } /** diff --git a/aws/src/test/java/org/apache/iceberg/aws/glue/GlueToIcebergConverterTest.java b/aws/src/test/java/org/apache/iceberg/aws/glue/GlueToIcebergConverterTest.java index 88a9eee2942b..40365f36d353 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/glue/GlueToIcebergConverterTest.java +++ b/aws/src/test/java/org/apache/iceberg/aws/glue/GlueToIcebergConverterTest.java @@ -26,9 +26,11 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.junit.Test; import software.amazon.awssdk.services.glue.model.Database; +import software.amazon.awssdk.services.glue.model.DatabaseInput; import software.amazon.awssdk.services.glue.model.Table; public class GlueToIcebergConverterTest { @@ -89,4 +91,41 @@ public void validateTable_icebergPropertyValueWrong() { () -> GlueToIcebergConverter.validateTable(table, "name") ); } + + @Test + public void testToDatabaseInput() { + Map properties = ImmutableMap.of( + IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description", + IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://location", + "key", "val"); + DatabaseInput databaseInput = IcebergToGlueConverter.toDatabaseInput(Namespace.of("ns"), properties); + Assert.assertEquals("Location should be set", "s3://location", databaseInput.locationUri()); + Assert.assertEquals("Description should be set", "description", databaseInput.description()); + Assert.assertEquals("Parameters should be set", ImmutableMap.of("key", "val"), databaseInput.parameters()); + Assert.assertEquals("Database name should be set", "ns", databaseInput.name()); + } + + @Test + public void testToDatabaseInputEmptyLocation() { + Map properties = ImmutableMap.of( + IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description", + "key", "val"); + DatabaseInput databaseInput = IcebergToGlueConverter.toDatabaseInput(Namespace.of("ns"), properties); + Assert.assertNull("Location should not be set", databaseInput.locationUri()); + Assert.assertEquals("Description should be set", "description", databaseInput.description()); + Assert.assertEquals("Parameters should be set", ImmutableMap.of("key", "val"), databaseInput.parameters()); + Assert.assertEquals("Database name should be set", "ns", databaseInput.name()); + } + + @Test + public void testToDatabaseInputEmptyDescription() { + Map properties = ImmutableMap.of( + IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://location", + "key", "val"); + DatabaseInput databaseInput = IcebergToGlueConverter.toDatabaseInput(Namespace.of("ns"), properties); + Assert.assertEquals("Location should be set", "s3://location", databaseInput.locationUri()); + Assert.assertNull("Description should not be set", databaseInput.description()); + Assert.assertEquals("Parameters should be set", ImmutableMap.of("key", "val"), databaseInput.parameters()); + Assert.assertEquals("Database name should be set", "ns", databaseInput.name()); + } }