Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add generic AWS S3 domain support #1470

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 22 additions & 16 deletions api/src/main/java/io/minio/BucketArgs.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@

package io.minio;

import io.minio.http.HttpUtils;
import io.minio.org.apache.commons.validator.routines.InetAddressValidator;
import java.util.Objects;
import java.util.regex.Pattern;

/** Base argument class holds bucket name and region. */
public abstract class BucketArgs extends BaseArgs {
Expand All @@ -34,32 +37,35 @@ public String region() {
/** Base argument builder class for {@link BucketArgs}. */
public abstract static class Builder<B extends Builder<B, A>, A extends BucketArgs>
extends BaseArgs.Builder<B, A> {
private static final Pattern BUCKET_NAME_REGEX =
Pattern.compile("^[a-z0-9][a-z0-9\\.\\-]{1,61}[a-z0-9]$");

protected void validateBucketName(String name) {
validateNotNull(name, "bucket name");

// Bucket names cannot be no less than 3 and no more than 63 characters long.
if (name.length() < 3 || name.length() > 63) {
if (!BUCKET_NAME_REGEX.matcher(name).find()) {
throw new IllegalArgumentException(
name + " : " + "bucket name must be at least 3 and no more than 63 characters long");
"bucket name '"
+ name
+ "' does not follow Amazon S3 standards. For more information refer "
+ "https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html");
}
// Successive periods in bucket names are not allowed.
if (name.contains("..")) {
String msg =
"bucket name cannot contain successive periods. For more information refer "
+ "http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html";
throw new IllegalArgumentException(name + " : " + msg);

if (InetAddressValidator.getInstance().isValidInet4Address(name)) {
throw new IllegalArgumentException(
"bucket name '" + name + "' must not be formatted as an IP address");
}
// Bucket names should be dns compatible.
if (!name.matches("^[a-z0-9][a-z0-9\\.\\-]+[a-z0-9]$")) {
String msg =
"bucket name does not follow Amazon S3 standards. For more information refer "
+ "http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html";
throw new IllegalArgumentException(name + " : " + msg);

if (name.contains("..") || name.contains(".-") || name.contains("-.")) {
throw new IllegalArgumentException(
"bucket name '" + name + "' cannot contain successive characters '..', '.-' and '-.'");
}
}

private void validateRegion(String region) {
validateNullOrNotEmptyString(region, "region");
if (region != null && !HttpUtils.REGION_REGEX.matcher(region).find()) {
throw new IllegalArgumentException("invalid region " + region);
}
}

@Override
Expand Down
148 changes: 65 additions & 83 deletions api/src/main/java/io/minio/MinioAsyncClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import java.nio.file.StandardOpenOption;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedList;
Expand All @@ -75,6 +76,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
Expand Down Expand Up @@ -131,22 +133,20 @@
public class MinioAsyncClient extends S3Base {
private MinioAsyncClient(
HttpUrl baseUrl,
String region,
boolean isAwsHost,
boolean isFipsHost,
boolean isAccelerateHost,
boolean isDualStackHost,
String awsS3Prefix,
String awsDomainSuffix,
boolean awsDualstack,
boolean useVirtualStyle,
String region,
Provider provider,
OkHttpClient httpClient) {
super(
baseUrl,
region,
isAwsHost,
isFipsHost,
isAccelerateHost,
isDualStackHost,
awsS3Prefix,
awsDomainSuffix,
awsDualstack,
useVirtualStyle,
region,
provider,
httpClient);
}
Expand Down Expand Up @@ -3213,87 +3213,66 @@ public static Builder builder() {
/** Argument builder of {@link MinioClient}. */
public static final class Builder {
private HttpUrl baseUrl;
private String region;
private boolean isAwsHost;
private boolean isFipsHost;
private boolean isAccelerateHost;
private boolean isDualStackHost;
private String awsS3Prefix;
private String awsDomainSuffix;
private boolean awsDualstack;
private boolean useVirtualStyle;

private String region;
private Provider provider;
private OkHttpClient httpClient;

private boolean isAwsChinaHost;
private String regionInUrl;
private void setAwsInfo(String host, boolean https) {
this.awsS3Prefix = null;
this.awsDomainSuffix = null;
this.awsDualstack = false;

private boolean isAwsFipsEndpoint(String endpoint) {
return endpoint.startsWith("s3-fips.");
}
if (!HttpUtils.HOSTNAME_REGEX.matcher(host).find()) return;

private boolean isAwsAccelerateEndpoint(String endpoint) {
return endpoint.startsWith("s3-accelerate.");
}
if (HttpUtils.AWS_ELB_ENDPOINT_REGEX.matcher(host).find()) {
String[] tokens = host.split("\\.elb\\.amazonaws\\.com", 1)[0].split("\\.");
this.region = tokens[tokens.length - 1];
return;
}

private boolean isAwsEndpoint(String endpoint) {
return (endpoint.startsWith("s3.")
|| isAwsFipsEndpoint(endpoint)
|| isAwsAccelerateEndpoint(endpoint))
&& (endpoint.endsWith(".amazonaws.com") || endpoint.endsWith(".amazonaws.com.cn"));
}
if (!HttpUtils.AWS_ENDPOINT_REGEX.matcher(host).find()) return;

private boolean isAwsDualStackEndpoint(String endpoint) {
return endpoint.contains(".dualstack.");
}
if (!HttpUtils.AWS_S3_ENDPOINT_REGEX.matcher(host).find()) {
throw new IllegalArgumentException("invalid Amazon AWS host " + host);
}

Matcher matcher = HttpUtils.AWS_S3_PREFIX_REGEX.matcher(host);
matcher.lookingAt();
int end = matcher.end();

this.awsS3Prefix = host.substring(0, end);
if (this.awsS3Prefix.contains("s3-accesspoint") && !https) {
throw new IllegalArgumentException("use HTTPS scheme for host " + host);
}

/**
* Extracts region from AWS endpoint if available. Region is placed at second token normal
* endpoints and third token for dualstack endpoints.
*
* <p>Region is marked in square brackets in below examples.
* <pre>
* https://s3.[us-east-2].amazonaws.com
* https://s3.dualstack.[ca-central-1].amazonaws.com
* https://s3.[cn-north-1].amazonaws.com.cn
* https://s3.dualstack.[cn-northwest-1].amazonaws.com.cn
*/
private String extractRegion(String endpoint) {
String[] tokens = endpoint.split("\\.");
String token = tokens[1];

// If token is "dualstack", then region might be in next token.
if (token.equals("dualstack")) {
token = tokens[2];
String[] tokens = host.substring(end).split("\\.");
awsDualstack = "dualstack".equals(tokens[0]);
if (awsDualstack) tokens = Arrays.copyOfRange(tokens, 1, tokens.length);
String regionInHost = null;
if (!tokens[0].equals("vpce") && !tokens[0].equals("amazonaws")) {
regionInHost = tokens[0];
tokens = Arrays.copyOfRange(tokens, 1, tokens.length);
}
this.awsDomainSuffix = String.join(".", tokens);

// If token is equal to "amazonaws", region is not passed in the endpoint.
if (token.equals("amazonaws")) {
return null;
if (host.equals("s3-external-1.amazonaws.com")) regionInHost = "us-east-1";
if (host.equals("s3-us-gov-west-1.amazonaws.com")
|| host.equals("s3-fips-us-gov-west-1.amazonaws.com")) {
regionInHost = "us-gov-west-1";
}

// Return token as region.
return token;
if (regionInHost != null) this.region = regionInHost;
}

private void setBaseUrl(HttpUrl url) {
String host = url.host();

this.isAwsHost = isAwsEndpoint(host);
this.isAwsChinaHost = false;
if (this.isAwsHost) {
this.isAwsChinaHost = host.endsWith(".cn");
url =
url.newBuilder()
.host(this.isAwsChinaHost ? "amazonaws.com.cn" : "amazonaws.com")
.build();
this.isFipsHost = isAwsFipsEndpoint(host);
this.isAccelerateHost = isAwsAccelerateEndpoint(host);
this.isDualStackHost = isAwsDualStackEndpoint(host);
this.regionInUrl = extractRegion(host);
this.useVirtualStyle = true;
} else {
this.useVirtualStyle = host.endsWith("aliyuncs.com");
}

this.baseUrl = url;
this.setAwsInfo(url.host(), url.isHttps());
this.useVirtualStyle = this.awsDomainSuffix != null || url.host().endsWith("aliyuncs.com");
}

public Builder endpoint(String endpoint) {
Expand Down Expand Up @@ -3325,8 +3304,9 @@ public Builder endpoint(HttpUrl url) {
}

public Builder region(String region) {
HttpUtils.validateNullOrNotEmptyString(region, "region");
this.regionInUrl = region;
if (region != null && !HttpUtils.REGION_REGEX.matcher(region).find()) {
throw new IllegalArgumentException("invalid region " + region);
}
this.region = region;
return this;
}
Expand All @@ -3349,7 +3329,11 @@ public Builder httpClient(OkHttpClient httpClient) {

public MinioAsyncClient build() {
HttpUtils.validateNotNull(this.baseUrl, "endpoint");
if (this.isAwsChinaHost && this.regionInUrl == null && this.region == null) {

if (this.awsDomainSuffix != null
&& this.awsDomainSuffix.endsWith(".cn")
&& !this.awsS3Prefix.endsWith("s3-accelerate.")
&& this.region == null) {
throw new IllegalArgumentException(
"Region missing in Amazon S3 China endpoint " + this.baseUrl);
}
Expand All @@ -3358,17 +3342,15 @@ public MinioAsyncClient build() {
this.httpClient =
HttpUtils.newDefaultHttpClient(
DEFAULT_CONNECTION_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT);
if (this.region == null) this.region = regionInUrl;
}

return new MinioAsyncClient(
baseUrl,
region,
isAwsHost,
isFipsHost,
isAccelerateHost,
isDualStackHost,
awsS3Prefix,
awsDomainSuffix,
awsDualstack,
useVirtualStyle,
region,
provider,
httpClient);
}
Expand Down
19 changes: 17 additions & 2 deletions api/src/main/java/io/minio/MinioClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -2405,12 +2405,22 @@ public void traceOff() throws IOException {
asyncClient.traceOff();
}

/** Enables accelerate endpoint for Amazon S3 endpoint. */
/**
* Enables accelerate endpoint for Amazon S3 endpoint.
*
* @deprecated This method is no longer supported.
*/
@Deprecated
public void enableAccelerateEndpoint() {
asyncClient.enableAccelerateEndpoint();
}

/** Disables accelerate endpoint for Amazon S3 endpoint. */
/**
* Disables accelerate endpoint for Amazon S3 endpoint.
*
* @deprecated This method is no longer supported.
*/
@Deprecated
public void disableAccelerateEndpoint() {
asyncClient.disableAccelerateEndpoint();
}
Expand All @@ -2435,6 +2445,11 @@ public void disableVirtualStyleEndpoint() {
asyncClient.disableVirtualStyleEndpoint();
}

/** Sets AWS S3 domain prefix. */
public void setAwsS3Prefix(String awsS3Prefix) {
asyncClient.setAwsS3Prefix(awsS3Prefix);
}

public static Builder builder() {
return new Builder();
}
Expand Down
Loading