Skip to content

AWS: support setting location and description for Glue database #3467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jackye1995 merged 1 commit into from
Nov 4, 2021
Merged

AWS: support setting location and description for Glue database #3467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions aws/src/integration/java/org/apache/iceberg/aws/glue/GlueCatalogNamespaceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
Expand All @@ -48,9 +49,19 @@ public void testCreateNamespace() {
EntityNotFoundException.class,
"not found",
() -> glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()));
glueCatalog.createNamespace(Namespace.of(namespace));
Map<String, String> properties = ImmutableMap.of(
IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description",
IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://location",
"key", "val");
Namespace ns = Namespace.of(namespace);
glueCatalog.createNamespace(ns, properties);
Database database = glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()).database();
Assert.assertEquals("namespace must equal database name", namespace, database.name());
Assert.assertEquals("namespace description should be set", "description", database.description());
Assert.assertEquals("namespace location should be set", "s3://location", database.locationUri());
Assert.assertEquals("namespace parameters should be set",
ImmutableMap.of("key", "val"), database.parameters());
Assert.assertEquals(properties, glueCatalog.loadNamespaceMetadata(ns));
}

@Test
Expand Down Expand Up @@ -100,27 +111,38 @@ public void testNamespaceProperties() {
Map<String, String> properties = Maps.newHashMap();
properties.put("key", "val");
properties.put("key2", "val2");
properties.put(IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://test");
properties.put(IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description");
glueCatalog.setProperties(Namespace.of(namespace), properties);
Database database = glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()).database();
Assert.assertTrue(database.parameters().containsKey("key"));
Assert.assertEquals("val", database.parameters().get("key"));
Assert.assertTrue(database.parameters().containsKey("key2"));
Assert.assertEquals("val2", database.parameters().get("key2"));
Assert.assertEquals("s3://test", database.locationUri());
Assert.assertEquals("description", database.description());
// remove properties
glueCatalog.removeProperties(Namespace.of(namespace), Sets.newHashSet("key"));
glueCatalog.removeProperties(Namespace.of(namespace), Sets.newHashSet(
"key", IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY));
database = glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()).database();
Assert.assertFalse(database.parameters().containsKey("key"));
Assert.assertTrue(database.parameters().containsKey("key2"));
Assert.assertEquals("val2", database.parameters().get("key2"));
Assert.assertNull(database.locationUri());
Assert.assertNull(database.description());
// add back
properties = Maps.newHashMap();
properties.put("key", "val");
properties.put(IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://test2");
properties.put(IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description2");
glueCatalog.setProperties(Namespace.of(namespace), properties);
database = glue.getDatabase(GetDatabaseRequest.builder().name(namespace).build()).database();
Assert.assertTrue(database.parameters().containsKey("key"));
Assert.assertEquals("val", database.parameters().get("key"));
Assert.assertTrue(database.parameters().containsKey("key2"));
Assert.assertEquals("val2", database.parameters().get("key2"));
Assert.assertEquals("s3://test2", database.locationUri());
Assert.assertEquals("description2", database.description());
}

@Test
Expand Down
17 changes: 14 additions & 3 deletions aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import software.amazon.awssdk.services.glue.GlueClient;
import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest;
import software.amazon.awssdk.services.glue.model.CreateTableRequest;
import software.amazon.awssdk.services.glue.model.Database;
import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest;
import software.amazon.awssdk.services.glue.model.DeleteTableRequest;
import software.amazon.awssdk.services.glue.model.EntityNotFoundException;
Expand Down Expand Up @@ -333,11 +334,21 @@ public List<Namespace> listNamespaces(Namespace namespace) throws NoSuchNamespac
public Map<String, String> loadNamespaceMetadata(Namespace namespace) throws NoSuchNamespaceException {
String databaseName = IcebergToGlueConverter.toDatabaseName(namespace);
try {
GetDatabaseResponse response = glue.getDatabase(GetDatabaseRequest.builder()
Database database = glue.getDatabase(GetDatabaseRequest.builder()
.catalogId(awsProperties.glueCatalogId())
.name(databaseName)
.build());
Map<String, String> result = response.database().parameters();
.build())
.database();
Map<String, String> result = Maps.newHashMap(database.parameters());

if (database.locationUri() != null) {
result.put(IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, database.locationUri());
}

if (database.description() != null) {
result.put(IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, database.description());
}

LOG.debug("Loaded metadata for namespace {} found {}", namespace, result);
return result;
} catch (InvalidInputException e) {
Expand Down
20 changes: 16 additions & 4 deletions aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.TypeUtil;
Expand All @@ -55,6 +56,8 @@ private IcebergToGlueConverter() {

private static final Pattern GLUE_DB_PATTERN = Pattern.compile("^[a-z0-9_]{1,252}$");
private static final Pattern GLUE_TABLE_PATTERN = Pattern.compile("^[a-z0-9_]{1,255}$");
public static final String GLUE_DB_LOCATION_KEY = "location";
public static final String GLUE_DB_DESCRIPTION_KEY = "comment";
public static final String ICEBERG_FIELD_USAGE = "iceberg.field.usage";
public static final String ICEBERG_FIELD_TYPE_TYPE_ID = "iceberg.field.type.typeid";
public static final String ICEBERG_FIELD_TYPE_STRING = "iceberg.field.type.string";
Expand Down Expand Up @@ -118,10 +121,19 @@ static String getDatabaseName(TableIdentifier tableIdentifier) {
* @return Glue DatabaseInput
*/
static DatabaseInput toDatabaseInput(Namespace namespace, Map<String, String> metadata) {
return DatabaseInput.builder()
.name(toDatabaseName(namespace))
.parameters(metadata)
.build();
DatabaseInput.Builder builder = DatabaseInput.builder().name(toDatabaseName(namespace));
Map<String, String> parameters = Maps.newHashMap();
metadata.forEach((k, v) -> {
if (GLUE_DB_DESCRIPTION_KEY.equals(k)) {
builder.description(v);
} else if (GLUE_DB_LOCATION_KEY.equals(k)) {
builder.locationUri(v);
} else {
parameters.put(k, v);
}
});

return builder.parameters(parameters).build();
}

/**
Expand Down
39 changes: 39 additions & 0 deletions aws/src/test/java/org/apache/iceberg/aws/glue/GlueToIcebergConverterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.junit.Assert;
import org.junit.Test;
import software.amazon.awssdk.services.glue.model.Database;
import software.amazon.awssdk.services.glue.model.DatabaseInput;
import software.amazon.awssdk.services.glue.model.Table;

public class GlueToIcebergConverterTest {
Expand Down Expand Up @@ -89,4 +91,41 @@ public void validateTable_icebergPropertyValueWrong() {
() -> GlueToIcebergConverter.validateTable(table, "name")
);
}

@Test
public void testToDatabaseInput() {
Map<String, String> properties = ImmutableMap.of(
IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description",
IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://location",
"key", "val");
DatabaseInput databaseInput = IcebergToGlueConverter.toDatabaseInput(Namespace.of("ns"), properties);
Assert.assertEquals("Location should be set", "s3://location", databaseInput.locationUri());
Assert.assertEquals("Description should be set", "description", databaseInput.description());
Assert.assertEquals("Parameters should be set", ImmutableMap.of("key", "val"), databaseInput.parameters());
Assert.assertEquals("Database name should be set", "ns", databaseInput.name());
}

@Test
public void testToDatabaseInputEmptyLocation() {
Map<String, String> properties = ImmutableMap.of(
IcebergToGlueConverter.GLUE_DB_DESCRIPTION_KEY, "description",
"key", "val");
DatabaseInput databaseInput = IcebergToGlueConverter.toDatabaseInput(Namespace.of("ns"), properties);
Assert.assertNull("Location should not be set", databaseInput.locationUri());
Assert.assertEquals("Description should be set", "description", databaseInput.description());
Assert.assertEquals("Parameters should be set", ImmutableMap.of("key", "val"), databaseInput.parameters());
Assert.assertEquals("Database name should be set", "ns", databaseInput.name());
}

@Test
public void testToDatabaseInputEmptyDescription() {
Map<String, String> properties = ImmutableMap.of(
IcebergToGlueConverter.GLUE_DB_LOCATION_KEY, "s3://location",
"key", "val");
DatabaseInput databaseInput = IcebergToGlueConverter.toDatabaseInput(Namespace.of("ns"), properties);
Assert.assertEquals("Location should be set", "s3://location", databaseInput.locationUri());
Assert.assertNull("Description should not be set", databaseInput.description());
Assert.assertEquals("Parameters should be set", ImmutableMap.of("key", "val"), databaseInput.parameters());
Assert.assertEquals("Database name should be set", "ns", databaseInput.name());
}
}