diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java index e5a23a9a571cc..f970423b336ac 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java @@ -271,19 +271,22 @@ public void createTable(String tableName, try { Map mapSchema = parquetSchemaToMapSchema(storageSchema, syncConfig.supportTimestamp, false); - List schemaPartitionKeys = new ArrayList<>(); List schemaWithoutPartitionKeys = new ArrayList<>(); for (String key : mapSchema.keySet()) { String keyType = getPartitionKeyType(mapSchema, key); Column column = new Column().withName(key).withType(keyType.toLowerCase()).withComment(""); // In Glue, the full schema should exclude the partition keys - if (syncConfig.partitionFields.contains(key)) { - schemaPartitionKeys.add(column); - } else { + if (!syncConfig.partitionFields.contains(key)) { schemaWithoutPartitionKeys.add(column); } } + // now create the schema partition + List schemaPartitionKeys = syncConfig.partitionFields.stream().map(partitionKey -> { + String keyType = getPartitionKeyType(mapSchema, partitionKey); + return new Column().withName(partitionKey).withType(keyType.toLowerCase()).withComment(""); + }).collect(Collectors.toList()); + StorageDescriptor storageDescriptor = new StorageDescriptor(); serdeProperties.put("serialization.format", "1"); storageDescriptor