[590] Add Iceberg Glue Catalog Sync implementation

pom.xml

@@ -51,6 +51,7 @@
         <module>xtable-hudi-support</module>
         <module>xtable-core</module>
         <module>xtable-utilities</module>
+        <module>xtable-aws</module>
     </modules>
 
     <properties>
@@ -61,8 +62,9 @@
         <junit.version>5.9.0</junit.version>
         <lombok.version>1.18.30</lombok.version>
         <lombok-maven-plugin.version>1.18.20.0</lombok-maven-plugin.version>
-        <hadoop.version>3.4.0</hadoop.version>
+        <hadoop.version>3.4.1</hadoop.version>
         <hudi.version>0.14.0</hudi.version>
+        <aws.version>2.29.40</aws.version>
         <maven-source-plugin.version>3.3.1</maven-source-plugin.version>
         <maven-javadoc-plugin.version>3.8.0</maven-javadoc-plugin.version>
         <maven-gpg-plugin.version>3.2.4</maven-gpg-plugin.version>
@@ -371,19 +373,26 @@
                 <version>${hadoop.version}</version>
                 <scope>runtime</scope>
             </dependency>
-            <dependency>
-                <groupId>com.amazonaws</groupId>
-                <artifactId>aws-java-sdk-bundle</artifactId>
-                <version>1.12.328</version>
-                <scope>runtime</scope>
-            </dependency>
             <dependency>
                 <groupId>com.google.cloud.bigdataoss</groupId>
                 <artifactId>gcs-connector</artifactId>
                 <version>hadoop3-2.2.22</version>
                 <scope>runtime</scope>
             </dependency>
 
+            <!-- AWS -->
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>bundle</artifactId>
+                <version>${aws.version}</version>
+                <scope>runtime</scope>
+            </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>glue</artifactId>
+                <version>${aws.version}</version>
+            </dependency>
+
             <!-- Protobuf -->
             <dependency>
                 <groupId>com.google.protobuf</groupId>

rfc/template.md

@@ -0,0 +1,55 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+# RFC-[number]: [Title]
+
+## Proposers
+
+- @<proposer1 github username>
+- @<proposer2 github username>
+
+## Approvers
+- @<approver1 github username>
+- @<approver2 github username>
+
+## Status
+
+GH Feature Request: <link to umbrella JIRA>
+
+> Please keep the status updated in `rfc/README.md`.
+
+## Abstract
+
+Describe the problem you are trying to solve and a brief description of why it’s needed.
+
+## Background
+Introduce any background context which is relevant or necessary to understand the feature and design choices.
+
+## Implementation
+Describe the new thing you want to do in appropriate detail, how it fits into the project architecture.<br>
+Provide a detailed description of how you intend to implement this feature, this may be fairly extensive and have large subsections of its own or it may be a few sentences.<br>
+Use judgement to decide on how detailed the description needs to be based on the scope of the change. If unclear, you can ask questions in [email protected].
+
+## Rollout/Adoption Plan
+
+- Are there any breaking changes as part of this new feature/functionality?
+- What impact (if any) will there be on existing users?
+- If we are changing behavior how will we phase out the older behavior? When will we remove the existing behavior? 
+- If we need special migration tools, describe them here.
+
+## Test Plan
+
+Describe in few sentences how the RFC will be tested. How will we know that the implementation works as expected? How will we know nothing breaks?

xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java

@@ -61,5 +61,5 @@ public class ExternalCatalogConfig {
   /**
    * The properties for this catalog, used for providing any custom behaviour during catalog sync
    */
-  @NonNull @Builder.Default Map<String, String> catalogProperties = Collections.emptyMap();
+  @Builder.Default Map<String, String> catalogProperties = Collections.emptyMap();
 }

xtable-api/src/main/java/org/apache/xtable/model/exception/ErrorCode.java

@@ -31,7 +31,8 @@ public enum ErrorCode {
   UNSUPPORTED_SCHEMA_TYPE(10007),
   UNSUPPORTED_FEATURE(10008),
   PARSE_EXCEPTION(10009),
-  CATALOG_REFRESH_EXCEPTION(10010);
+  CATALOG_REFRESH_EXCEPTION(10010),
+  CATALOG_SYNC_GENERIC_EXCEPTION(10011);
 
   private final int errorCode;
 

xtable-api/src/main/java/org/apache/xtable/model/storage/CatalogType.java

@@ -25,4 +25,5 @@
  */
 public class CatalogType {
   public static final String STORAGE = "STORAGE";
+  public static final String GLUE = "GLUE";
 }

xtable-api/src/main/java/org/apache/xtable/spi/sync/CatalogSync.java

@@ -83,6 +83,11 @@ private <TABLE> CatalogSyncStatus syncCatalog(
       CatalogSyncClient<TABLE> catalogSyncClient,
       CatalogTableIdentifier tableIdentifier,
       InternalTable table) {
+    log.info(
+        "Running catalog sync for table {} with format {} using catalogSync {}",
+        table.getBasePath(),
+        table.getTableFormat(),
+        catalogSyncClient.getClass().getName());
     if (!catalogSyncClient.hasDatabase(tableIdentifier)) {
       catalogSyncClient.createDatabase(tableIdentifier);
     }

xtable-api/src/test/java/org/apache/xtable/model/catalog/TestHierarchicalTableIdentifier.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.model.catalog;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class TestHierarchicalTableIdentifier {
+
+  @Test
+  void testToString() {
+    HierarchicalTableIdentifier catalogTableIdentifier =
+        ThreePartHierarchicalTableIdentifier.fromDotSeparatedIdentifier(
+            "catalogName.databaseName.tableName");
+    assertEquals("catalogName.databaseName.tableName", catalogTableIdentifier.getId());
+  }
+
+  @Test
+  void testConstructorForHierarchicalTableIdentifier() {
+    Assertions.assertDoesNotThrow(
+        () ->
+            ThreePartHierarchicalTableIdentifier.fromDotSeparatedIdentifier(
+                "catalogName.databaseName.tableName"));
+    Assertions.assertDoesNotThrow(
+        () ->
+            ThreePartHierarchicalTableIdentifier.fromDotSeparatedIdentifier(
+                "databaseName.tableName"));
+    Assertions.assertThrows(
+        IllegalArgumentException.class,
+        () -> ThreePartHierarchicalTableIdentifier.fromDotSeparatedIdentifier("tableName"));
+  }
+}

xtable-aws/pom.xml

@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.xtable</groupId>
+        <artifactId>xtable</artifactId>
+        <version>0.2.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>xtable-aws</artifactId>
+    <name>XTable AWS</name>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.xtable</groupId>
+            <artifactId>xtable-core_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.xtable</groupId>
+            <artifactId>xtable-core_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <classifier>tests</classifier>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Hadoop dependencies -->
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- AWS Glue dependencies -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>glue</artifactId>
+        </dependency>
+
+        <!-- Junit -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Mockito -->
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <version>${maven-deploy-plugin.version}</version>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

xtable-aws/src/main/java/org/apache/xtable/glue/DefaultGlueClientFactory.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.glue;
+
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.xtable.exception.ConfigurationException;
+import org.apache.xtable.reflection.ReflectionUtils;
+
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.glue.GlueClientBuilder;
+
+/**
+ * Factory class for creating and configuring instances of {@link GlueClient} with settings provided
+ * by {@link GlueCatalogConfig}.
+ *
+ * <p>This factory is responsible for setting the AWS region and credentials for the Glue client. If
+ * a custom credentials provider class is specified in {@code GlueCatalogConfig}, it will use
+ * reflection to instantiate the provider; otherwise, it defaults to the standard AWS credentials
+ * provider.
+ */
+public class DefaultGlueClientFactory extends GlueClientFactory {
+
+  public DefaultGlueClientFactory(GlueCatalogConfig glueConfig) {
+    super(glueConfig);
+  }
+
+  public GlueClient getGlueClient() {
+    GlueClientBuilder builder = GlueClient.builder();
+    if (!StringUtils.isEmpty(glueConfig.getRegion())) {
+      builder.region(Region.of(glueConfig.getRegion()));
+    }
+
+    AwsCredentialsProvider credentialsProvider;
+    if (!StringUtils.isEmpty(glueConfig.getClientCredentialsProviderClass())) {
+      String className = glueConfig.getClientCredentialsProviderClass();
+      try {
+        credentialsProvider =
+            ReflectionUtils.createInstanceOfClassFromStaticMethod(
+                className,
+                "create",
+                new Class<?>[] {Map.class},
+                new Object[] {glueConfig.getClientCredentialConfigs()});
+      } catch (ConfigurationException e) {
+        // retry credentialsProvider creation without arguments if not a ClassNotFoundException
+        if (e.getCause() instanceof ClassNotFoundException) {
+          throw e;
+        }
+        credentialsProvider =
+            ReflectionUtils.createInstanceOfClassFromStaticMethod(className, "create");
+      }
+    } else {
+      credentialsProvider = DefaultCredentialsProvider.create();
+    }
+
+    builder.credentialsProvider(credentialsProvider);
+    return builder.build();
+  }
+}