diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 7b4376b04ce4b..27d9e5680f910 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -117,6 +117,9 @@ jobs: if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI run: mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + - name: Test runtime breakdown + run: + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr test-hudi-hadoop-mr-and-hudi-java-client: runs-on: ubuntu-latest @@ -151,6 +154,9 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} run: mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS + - name: Test runtime breakdown + run: + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr test-spark-java17: runs-on: ubuntu-latest @@ -208,6 +214,9 @@ jobs: if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI run: mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS + - name: Test runtime breakdown + run: + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr test-flink: runs-on: ubuntu-latest diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index 85d185fbc2c5c..759cfa4adfeca 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -143,6 +143,9 @@ stages: - script: | grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr + displayName: Test runtime breakdown - job: UT_FT_2 displayName: FT client/spark-client timeoutInMinutes: '150' @@ -167,6 +170,9 @@ stages: - script: | grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr + displayName: Test runtime breakdown - job: UT_FT_3 displayName: UT spark-datasource timeoutInMinutes: '240' @@ -191,6 +197,9 @@ stages: - script: | grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr + displayName: Test runtime breakdown - job: UT_FT_4 displayName: UT FT other modules timeoutInMinutes: '240' @@ -224,3 +233,6 @@ stages: - script: | grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr + displayName: Test runtime breakdown diff --git a/hudi-aws/pom.xml b/hudi-aws/pom.xml index 682b73ecacd77..30f2d892d2f9b 100644 --- a/hudi-aws/pom.xml +++ b/hudi-aws/pom.xml @@ -186,12 +186,6 @@ ${aws.sdk.httpcore.version} - - software.amazon.awssdk - sts - ${aws.sdk.version} - - org.apache.hudi diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java index 97df83929e9cf..4342a529d29a0 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieAWSCredentialsProviderFactory.java @@ -36,9 +36,6 @@ public static AwsCredentialsProvider getAwsCredentialsProvider(Properties props) private static AwsCredentialsProvider getAwsCredentialsProviderChain(Properties props) { List providers = new ArrayList<>(); - if (HoodieConfigAWSAssumedRoleCredentialsProvider.validConf(props)) { - providers.add(new HoodieConfigAWSAssumedRoleCredentialsProvider(props)); - } HoodieConfigAWSCredentialsProvider hoodieConfigAWSCredentialsProvider = new HoodieConfigAWSCredentialsProvider(props); if (hoodieConfigAWSCredentialsProvider.resolveCredentials() != null) { providers.add(hoodieConfigAWSCredentialsProvider); diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java deleted file mode 100644 index 89c31b8a08b30..0000000000000 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hudi.aws.credentials; - -import org.apache.hudi.common.util.StringUtils; -import org.apache.hudi.config.HoodieAWSConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; -import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; - -import java.util.Properties; - -/** - * Credentials provider which assumes AWS role from Hoodie config and fetches its credentials. - */ -public class HoodieConfigAWSAssumedRoleCredentialsProvider implements AwsCredentialsProvider { - - private static final Logger LOG = LoggerFactory.getLogger(HoodieConfigAWSAssumedRoleCredentialsProvider.class); - - private final StsAssumeRoleCredentialsProvider credentialsProvider; - - public HoodieConfigAWSAssumedRoleCredentialsProvider(Properties props) { - String roleArn = props.getProperty(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN.key()); - AssumeRoleRequest req = AssumeRoleRequest.builder() - .roleArn(roleArn) - .roleSessionName("hoodie") - .build(); - StsClient stsClient = StsClient.builder().build(); - - this.credentialsProvider = StsAssumeRoleCredentialsProvider.builder() - .stsClient(stsClient) - .refreshRequest(req) - .build(); - } - - public static boolean validConf(Properties props) { - String roleArn = props.getProperty(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN.key()); - return !StringUtils.isNullOrEmpty(roleArn); - } - - @Override - public AwsCredentials resolveCredentials() { - return credentialsProvider.resolveCredentials(); - } -} diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java index 4a846e4970f60..0e7609aba5cd8 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java @@ -81,7 +81,6 @@ import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS; import static org.apache.hudi.sync.common.util.TableUtils.tableId; -import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory; /** * This class implements all the AWS APIs to enable syncing of a Hudi Table with the @@ -108,9 +107,7 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient { public AWSGlueCatalogSyncClient(HiveSyncConfig config) { super(config); - this.awsGlue = GlueAsyncClient.builder() - .credentialsProvider(HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(config.getProps())) - .build(); + this.awsGlue = GlueAsyncClient.builder().build(); this.databaseName = config.getStringOrDefault(META_SYNC_DATABASE_NAME); this.skipTableArchive = config.getBooleanOrDefault(GlueCatalogSyncClientConfig.GLUE_SKIP_TABLE_ARCHIVE); this.enableMetadataTable = Boolean.toString(config.getBoolean(GLUE_METADATA_FILE_LISTING)).toUpperCase(); diff --git a/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java b/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java index a205dc94b30ff..45d6878fa3df4 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java +++ b/hudi-aws/src/main/java/org/apache/hudi/config/HoodieAWSConfig.java @@ -46,7 +46,7 @@ @ConfigClassProperty(name = "Amazon Web Services Configs", groupName = ConfigGroups.Names.AWS, description = "Amazon Web Services configurations to access resources like Amazon DynamoDB (for locks)," - + " Amazon CloudWatch (metrics) and Amazon Glue (metadata).") + + " Amazon CloudWatch (metrics).") public class HoodieAWSConfig extends HoodieConfig { public static final ConfigProperty AWS_ACCESS_KEY = ConfigProperty .key("hoodie.aws.access.key") @@ -69,13 +69,6 @@ public class HoodieAWSConfig extends HoodieConfig { .sinceVersion("0.10.0") .withDocumentation("AWS session token"); - public static final ConfigProperty AWS_ASSUME_ROLE_ARN = ConfigProperty - .key("hoodie.aws.role.arn") - .noDefaultValue() - .markAdvanced() - .sinceVersion("0.13.2") - .withDocumentation("AWS Role ARN to assume"); - private HoodieAWSConfig() { super(); } @@ -96,10 +89,6 @@ public String getAWSSessionToken() { return getString(AWS_SESSION_TOKEN); } - public String getAWSAssumeRoleARN() { - return getString(AWS_ASSUME_ROLE_ARN); - } - public static class Builder { private final HoodieAWSConfig awsConfig = new HoodieAWSConfig(); @@ -131,11 +120,6 @@ public HoodieAWSConfig.Builder withSessionToken(String sessionToken) { return this; } - public HoodieAWSConfig.Builder withAssumeRoleARN(String assumeRoleARN) { - awsConfig.setValue(AWS_ASSUME_ROLE_ARN, assumeRoleARN); - return this; - } - public Builder withDynamoDBTable(String dynamoDbTableName) { awsConfig.setValue(DYNAMODB_LOCK_TABLE_NAME, dynamoDbTableName); return this; diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java b/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java index d65f32109c124..7a5e776db8d53 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/TestHoodieAWSCredentialsProviderFactory.java @@ -39,20 +39,4 @@ public void testGetAWSCredentials() { assertEquals("random-secret-key", credentials.secretAccessKey()); assertEquals("random-session-token", credentials.sessionToken()); } - - @Test - public void testGetAWSCredentialsWithInvalidAssumeRole() { - // This test is to ensure that the AWS credentials provider factory fallbacks to default credentials - // when the assume role ARN is invalid. - HoodieConfig cfg = new HoodieConfig(); - cfg.setValue(HoodieAWSConfig.AWS_ACCESS_KEY, "random-access-key"); - cfg.setValue(HoodieAWSConfig.AWS_SECRET_KEY, "random-secret-key"); - cfg.setValue(HoodieAWSConfig.AWS_SESSION_TOKEN, "random-session-token"); - cfg.setValue(HoodieAWSConfig.AWS_ASSUME_ROLE_ARN, "invalid-role-arn"); - AwsSessionCredentials credentials = (AwsSessionCredentials) org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory.getAwsCredentialsProvider(cfg.getProps()).resolveCredentials(); - assertEquals("random-access-key", credentials.accessKeyId()); - assertEquals("random-secret-key", credentials.secretAccessKey()); - assertEquals("random-session-token", credentials.sessionToken()); - } - }