[MINOR][DNM] Full test runtime 2

.github/workflows/bot.yml

@@ -117,6 +117,9 @@ jobs:
         if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
         run:
           mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+      - name: Test runtime breakdown
+        run:
+          grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
 
   test-hudi-hadoop-mr-and-hudi-java-client:
     runs-on: ubuntu-latest
@@ -151,6 +154,9 @@ jobs:
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
           mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS
+      - name: Test runtime breakdown
+        run:
+          grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
 
   test-spark-java17:
     runs-on: ubuntu-latest
@@ -208,6 +214,9 @@ jobs:
         if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI
         run:
           mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS
+      - name: Test runtime breakdown
+        run:
+          grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
 
   test-flink:
     runs-on: ubuntu-latest

azure-pipelines-20230430.yml

@@ -143,6 +143,9 @@ stages:
           - script: |
               grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
             displayName: Top 100 long-running testcases
+          - script: |
+              grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
+            displayName: Test runtime breakdown
       - job: UT_FT_2
         displayName: FT client/spark-client
         timeoutInMinutes: '150'
@@ -167,6 +170,9 @@ stages:
           - script: |
               grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
             displayName: Top 100 long-running testcases
+          - script: |
+              grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
+            displayName: Test runtime breakdown
       - job: UT_FT_3
         displayName: UT spark-datasource
         timeoutInMinutes: '240'
@@ -191,6 +197,9 @@ stages:
           - script: |
               grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
             displayName: Top 100 long-running testcases
+          - script: |
+              grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
+            displayName: Test runtime breakdown
       - job: UT_FT_4
         displayName: UT FT other modules
         timeoutInMinutes: '240'
@@ -224,3 +233,6 @@ stages:
           - script: |
               grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
             displayName: Top 100 long-running testcases
+          - script: |
+              grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr
+            displayName: Test runtime breakdown

hudi-aws/pom.xml

@@ -186,12 +186,6 @@
             <version>${aws.sdk.httpcore.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>sts</artifactId>
-            <version>${aws.sdk.version}</version>
-        </dependency>
-
         <!-- Test -->
         <dependency>
             <groupId>org.apache.hudi</groupId>

hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java

@@ -36,9 +36,6 @@ public static AwsCredentialsProvider getAwsCredentialsProvider(Properties props)
 
   private static AwsCredentialsProvider getAwsCredentialsProviderChain(Properties props) {
     List<AwsCredentialsProvider> providers = new ArrayList<>();
-    if (HoodieConfigAWSAssumedRoleCredentialsProvider.validConf(props)) {
-      providers.add(new HoodieConfigAWSAssumedRoleCredentialsProvider(props));
-    }
     HoodieConfigAWSCredentialsProvider hoodieConfigAWSCredentialsProvider = new HoodieConfigAWSCredentialsProvider(props);
     if (hoodieConfigAWSCredentialsProvider.resolveCredentials() != null) {
       providers.add(hoodieConfigAWSCredentialsProvider);

hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java

@@ -81,7 +81,6 @@
 import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME;
 import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS;
 import static org.apache.hudi.sync.common.util.TableUtils.tableId;
-import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory;
 
 /**
  * This class implements all the AWS APIs to enable syncing of a Hudi Table with the
@@ -108,9 +107,7 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient {
 
   public AWSGlueCatalogSyncClient(HiveSyncConfig config) {
     super(config);
-    this.awsGlue = GlueAsyncClient.builder()
-            .credentialsProvider(HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(config.getProps()))
-            .build();
+    this.awsGlue = GlueAsyncClient.builder().build();
     this.databaseName = config.getStringOrDefault(META_SYNC_DATABASE_NAME);
     this.skipTableArchive = config.getBooleanOrDefault(GlueCatalogSyncClientConfig.GLUE_SKIP_TABLE_ARCHIVE);
     this.enableMetadataTable = Boolean.toString(config.getBoolean(GLUE_METADATA_FILE_LISTING)).toUpperCase();

hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java

@@ -46,7 +46,7 @@
 @ConfigClassProperty(name = "Amazon Web Services Configs",
         groupName = ConfigGroups.Names.AWS,
         description = "Amazon Web Services configurations to access resources like Amazon DynamoDB (for locks),"
-            + " Amazon CloudWatch (metrics) and Amazon Glue (metadata).")
+            + " Amazon CloudWatch (metrics).")
 public class HoodieAWSConfig extends HoodieConfig {
   public static final ConfigProperty<String> AWS_ACCESS_KEY = ConfigProperty
       .key("hoodie.aws.access.key")
@@ -69,13 +69,6 @@ public class HoodieAWSConfig extends HoodieConfig {
       .sinceVersion("0.10.0")
       .withDocumentation("AWS session token");
 
-  public static final ConfigProperty<String> AWS_ASSUME_ROLE_ARN = ConfigProperty
-          .key("hoodie.aws.role.arn")
-          .noDefaultValue()
-          .markAdvanced()
-          .sinceVersion("0.13.2")
-          .withDocumentation("AWS Role ARN to assume");
-
   private HoodieAWSConfig() {
     super();
   }
@@ -96,10 +89,6 @@ public String getAWSSessionToken() {
     return getString(AWS_SESSION_TOKEN);
   }
 
-  public String getAWSAssumeRoleARN() {
-    return getString(AWS_ASSUME_ROLE_ARN);
-  }
-
   public static class Builder {
 
     private final HoodieAWSConfig awsConfig = new HoodieAWSConfig();
@@ -131,11 +120,6 @@ public HoodieAWSConfig.Builder withSessionToken(String sessionToken) {
       return this;
     }
 
-    public HoodieAWSConfig.Builder withAssumeRoleARN(String assumeRoleARN) {
-      awsConfig.setValue(AWS_ASSUME_ROLE_ARN, assumeRoleARN);
-      return this;
-    }
-
     public Builder withDynamoDBTable(String dynamoDbTableName) {
       awsConfig.setValue(DYNAMODB_LOCK_TABLE_NAME, dynamoDbTableName);
       return this;

hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java

@@ -39,20 +39,4 @@ public void testGetAWSCredentials() {
     assertEquals("random-secret-key", credentials.secretAccessKey());
     assertEquals("random-session-token", credentials.sessionToken());
   }
-
-  @Test
-  public void testGetAWSCredentialsWithInvalidAssumeRole() {
-    // This test is to ensure that the AWS credentials provider factory fallbacks to default credentials
-    // when the assume role ARN is invalid.
-    HoodieConfig cfg = new HoodieConfig();
-    cfg.setValue(HoodieAWSConfig.AWS_ACCESS_KEY, "random-access-key");
-    cfg.setValue(HoodieAWSConfig.AWS_SECRET_KEY, "random-secret-key");
-    cfg.setValue(HoodieAWSConfig.AWS_SESSION_TOKEN, "random-session-token");
-    cfg.setValue(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN, "invalid-role-arn");
-    AwsSessionCredentials credentials = (AwsSessionCredentials) org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(cfg.getProps()).resolveCredentials();
-    assertEquals("random-access-key", credentials.accessKeyId());
-    assertEquals("random-secret-key", credentials.secretAccessKey());
-    assertEquals("random-session-token", credentials.sessionToken());
-  }
-
 }