diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java index c47673ab7fa2..201dd6458758 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java @@ -65,7 +65,7 @@ public class GlueTestBase { static GlueCatalog glueCatalogWithSkip; static Schema schema = new Schema(Types.NestedField.required(1, "c1", Types.StringType.get(), "c1")); - static PartitionSpec partitionSpec = PartitionSpec.builderFor(schema).build(); + static PartitionSpec partitionSpec = PartitionSpec.builderFor(schema).identity("c1").build(); // table location properties static final Map tableLocationProperties = ImmutableMap.of( diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java index 8429e0f6953a..d40373016571 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java @@ -78,6 +78,7 @@ public void testCreateTable() { Assert.assertEquals("additionalLocations should match", tableLocationProperties.values().stream().sorted().collect(Collectors.toList()), response.table().storageDescriptor().additionalLocations().stream().sorted().collect(Collectors.toList())); + Assert.assertEquals(partitionSpec.fields().size(), response.table().partitionKeys().size()); // verify metadata file exists in S3 String metaLocation = response.table().parameters().get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP); String key = metaLocation.split(testBucketName, -1)[1].substring(1); diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java index ba71c191ca08..84d28aed41a8 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java +++ b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java @@ -209,7 +209,8 @@ static void setTableInputInformation(TableInput.Builder tableInputBuilder, Table .storageDescriptor(storageDescriptor .location(metadata.location()) .columns(toColumns(metadata)) - .build()); + .build()) + .partitionKeys(toPartitionColumns(metadata)); } catch (RuntimeException e) { LOG.warn("Encountered unexpected exception while converting Iceberg metadata to Glue table information", e); } @@ -302,4 +303,14 @@ private static void addColumnWithDedupe(List columns, Set dedupe dedupe.add(field.name()); } } + + private static List toPartitionColumns(TableMetadata metadata) { + return metadata.spec().fields().stream() + .map(f -> Column.builder() + .name(f.transform() + "(" + f.name() + ")") + .type(toTypeString(f.transform().getResultType(metadata.schema().findField(f.sourceId()).type()))) + .comment("fieldId: " + f.fieldId() + ", sourceId: " + f.sourceId() + ", transform: " + f.transform()) + .build()) + .collect(Collectors.toList()); + } } diff --git a/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java b/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java index ec67af4597cb..2c88f55ce335 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java +++ b/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java @@ -129,6 +129,7 @@ public void testSetTableInputInformation() { .identity("x") .withSpecId(1000) .build(); + TableMetadata tableMetadata = TableMetadata .newTableMetadata(schema, partitionSpec, "s3://test", tableLocationProperties); IcebergToGlueConverter.setTableInputInformation(actualTableInputBuilder, tableMetadata); @@ -161,6 +162,13 @@ public void testSetTableInputInformation() { )) .build())) .build()) + .partitionKeys(ImmutableList.of( + Column.builder() + .name("identity(x)") + .type("string") + .comment("fieldId: 1000, sourceId: 1, transform: identity") + .build() + )) .build(); Assert.assertEquals( @@ -175,6 +183,10 @@ public void testSetTableInputInformation() { "Columns should match", expectedTableInput.storageDescriptor().columns(), actualTableInput.storageDescriptor().columns()); + Assert.assertEquals( + "Partition keys should match", + expectedTableInput.partitionKeys(), + actualTableInput.partitionKeys()); } @Test