diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java index dcfc0a7ee8416..ed75477e12a66 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java @@ -64,6 +64,18 @@ public class STSClientFactory { private static final Logger LOG = LoggerFactory.getLogger(STSClientFactory.class); + /** + * The format of the STS Endpoint + */ + public static final String STS_ENDPOINT_URI_PATTERN = "^sts\\..*\\.amazonaws\\.com[.cn]*$"; + + /** + * Error thrown is the STS endpoint is not of the correct pattern. + */ + public static final String E_INVALID_STS_ENDPOINT_ERROR_MSG = + "Invalid STS Endpoint %s. The STS endpoint should match " + + STS_ENDPOINT_URI_PATTERN; + /** * Create the builder ready for any final configuration options. * Picks up connection settings from the Hadoop configuration, including @@ -125,6 +137,14 @@ public static StsClientBuilder builder( public static StsClientBuilder builder(final AwsCredentialsProvider credentials, final Configuration conf, final String stsEndpoint, final String stsRegion, final String bucket) throws IOException { + // If an STS endpoint is provided and if it is not STS_STANDARD (sts.amazonaws.com) + // it should match STS_ENDPOINT_URI_PATTERN. + if (!isEmpty(stsEndpoint) && + !STS_STANDARD.equals(stsEndpoint) && + !stsEndpoint.matches(STS_ENDPOINT_URI_PATTERN)) { + throw new IllegalArgumentException(String.format(E_INVALID_STS_ENDPOINT_ERROR_MSG,stsEndpoint)); + } + final StsClientBuilder stsClientBuilder = StsClient.builder(); Preconditions.checkArgument(credentials != null, "No credentials"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 290a4d995c757..4216369c20014 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -19,7 +19,6 @@ package org.apache.hadoop.fs.s3a; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.AccessDeniedException; import java.time.Duration; import java.time.OffsetDateTime; @@ -51,6 +50,7 @@ import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromSTSCredentials; import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.toAWSCredentials; import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.assertCredentialsEqual; +import static org.apache.hadoop.fs.s3a.auth.STSClientFactory.E_INVALID_STS_ENDPOINT_ERROR_MSG; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*; import static org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding.CREDENTIALS_CONVERTED_TO_DELEGATION_TOKEN; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -163,6 +163,119 @@ public void testSTS() throws IOException { } } + /** + * Create a StsClientBuilder instance with a given STS URL. + * + * @param stsURL The STS URL to be used to create an StsClientBuilder instance. + * @return StsClientBuilder. + * + * @throws IOException If there is an error while creating the StsClientBuilder instance. + */ + private StsClientBuilder createSTS(String stsURL, String region) throws IOException { + Configuration conf = getContract().getConf(); + S3AFileSystem testFS = getFileSystem(); + credentials = getS3AInternals().shareCredentials("testSTS"); + + String bucket = testFS.getBucket(); + return STSClientFactory.builder(conf, bucket, credentials, stsURL, region); + } + + /** + * Interface for creating a lambda function that verifies the exception that is thrown when creating + * an StsClientBuilder. + */ + @FunctionalInterface + interface VerifyCreateSTSException { + void verifyException(String stsURL) throws IOException; + } + + /** + * Test different STS URLs. We will test a combination of valid and invalid URLs. + * + * The property test.sts.endpoint can be set to point this at different + * STS endpoints. This test will use the AWS credentials (if provided) for + * S3A tests to request temporary credentials, then attempt to use those + * credentials instead. + * + * @throws IOException failure + */ + + @Test + public void testSTSURLs() throws IOException { + // List of STS URLs to check for validity. + String [] validStsURLs = { + "sts.amazonaws.com", + "sts.us-east-2.amazonaws.com", + "sts.us-east-1.amazonaws.com", + "sts.us-west-1.amazonaws.com", + "sts.us-west-2.amazonaws.com", + "sts.af-south-1.amazonaws.com", + "sts.ap-east-1.amazonaws.com", + "sts.ap-south-2.amazonaws.com", + "sts.ap-southeast-3.amazonaws.com", + "sts.ap-southeast-4.amazonaws.com", + "sts.ap-south-1.amazonaws.com", + "sts.ap-northeast-3.amazonaws.com", + "sts.ap-northeast-2.amazonaws.com", + "sts.ap-southeast-1.amazonaws.com", + "sts.ap-southeast-2.amazonaws.com", + "sts.ap-northeast-1.amazonaws.com", + "sts.ca-central-1.amazonaws.com", + "sts.ca-west-1.amazonaws.com", + "sts.cn-north-1.amazonaws.com.cn", + "sts.cn-northwest-1.amazonaws.com.cn", + "sts.eu-central-1.amazonaws.com", + "sts.eu-west-1.amazonaws.com", + "sts.eu-west-2.amazonaws.com", + "sts.eu-south-1.amazonaws.com", + "sts.eu-west-3.amazonaws.com", + "sts.eu-south-2.amazonaws.com", + "sts.eu-north-1.amazonaws.com", + "sts.eu-central-2.amazonaws.com", + "sts.il-central-1.amazonaws.com", + "sts.me-south-1.amazonaws.com", + "sts.me-central-1.amazonaws.com", + "sts.sa-east-1.amazonaws.com" + }; + + // The lambda function can run an invalid STS URL and verify + // that the correct exception is thrown. + VerifyCreateSTSException vcse = (stsURL) -> { + try { + createSTS(stsURL, "us-west-2"); + } catch (IllegalArgumentException iae) { + LOG.info("Expected Exception: {}", iae.toString()); + } + }; + + // INVALID STS URL VERIFICATIONS + + // STS URLs start with sts. + vcse.verifyException("invalid-sts.eu-west-2.com"); + + // Should have been sts.cn-northwest-1.amazonaws.com.cn. + vcse.verifyException("sts.cn-north-1.amazonaws.cn"); + + // Not a valid STS URL. + vcse.verifyException("www.google.com"); + + // Random string is not a valid STS URL. + vcse.verifyException("invalid-sts"); + + // VALID STS URL VERIFICATIONS + + // Verify that there are no exceptions when testing for the different STS URLs. + for (String validStsURL : validStsURLs) { + String [] splits = validStsURL.split("\\."); + if (validStsURL == "sts.amazonaws.com") { + createSTS(validStsURL, ""); + } + else { + createSTS(validStsURL, splits[1]); + } + } + } + protected String getStsEndpoint(final Configuration conf) { return conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT, DEFAULT_ASSUMED_ROLE_STS_ENDPOINT); @@ -327,14 +440,11 @@ public void testSessionCredentialsRegionBadEndpoint() throws Throwable { describe("Create a session with a bad region and expect fast failure"); IllegalArgumentException ex = expectedSessionRequestFailure( - IllegalArgumentException.class, + IllegalArgumentException.class, " ", EU_IRELAND, ""); LOG.info("Outcome: ", ex); - if (!(ex.getCause() instanceof URISyntaxException)) { - throw ex; - } } @Test