Add support for service-specific endpoint overrides without code changes.

core/aws-core/src/main/java/software/amazon/awssdk/awscore/endpoint/DefaultServiceEndpointBuilder.java

@@ -16,156 +16,71 @@
 package software.amazon.awssdk.awscore.endpoint;
 
 import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.function.Supplier;
 import software.amazon.awssdk.annotations.NotThreadSafe;
 import software.amazon.awssdk.annotations.SdkProtectedApi;
-import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.profiles.ProfileFile;
-import software.amazon.awssdk.profiles.ProfileFileSystemSetting;
-import software.amazon.awssdk.regions.EndpointTag;
 import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.regions.ServiceEndpointKey;
-import software.amazon.awssdk.regions.ServiceMetadata;
 import software.amazon.awssdk.regions.ServiceMetadataAdvancedOption;
-import software.amazon.awssdk.utils.Lazy;
-import software.amazon.awssdk.utils.Validate;
 
 /**
- * Uses service metadata and the request region to construct an endpoint for a specific service
+ * Uses service metadata and the request region to construct an endpoint for a specific service.
+ *
+ * @deprecated Use {@link AwsClientEndpointProvider}. This is only used by old client versions.
  */
 @NotThreadSafe
 @SdkProtectedApi
-// TODO We may not need this anymore, we should default to AWS partition when resolving
-// a region we don't know about yet.
+@Deprecated
 public final class DefaultServiceEndpointBuilder {
-    private final String serviceName;
-    private final String protocol;
-
+    private final AwsClientEndpointProvider.Builder endpointResolver = AwsClientEndpointProvider.builder();
     private Region region;
-    private Supplier<ProfileFile> profileFile;
-    private String profileName;
-    private final Map<ServiceMetadataAdvancedOption<?>, Object> advancedOptions = new HashMap<>();
-    private Boolean dualstackEnabled;
-    private Boolean fipsEnabled;
 
     public DefaultServiceEndpointBuilder(String serviceName, String protocol) {
-        this.serviceName = Validate.paramNotNull(serviceName, "serviceName");
-        this.protocol = Validate.paramNotNull(protocol, "protocol");
+        endpointResolver.serviceEndpointPrefix(serviceName)
+                        .defaultProtocol(protocol);
+    }
+
+    public Region getRegion() {
+        return region;
     }
 
     public DefaultServiceEndpointBuilder withRegion(Region region) {
-        if (region == null) {
-            throw new IllegalArgumentException("Region cannot be null");
-        }
         this.region = region;
+        endpointResolver.region(region);
         return this;
     }
 
     public DefaultServiceEndpointBuilder withProfileFile(Supplier<ProfileFile> profileFile) {
-        this.profileFile = profileFile;
+        endpointResolver.profileFile(profileFile);
         return this;
     }
 
     public DefaultServiceEndpointBuilder withProfileFile(ProfileFile profileFile) {
-        this.profileFile = () -> profileFile;
+        endpointResolver.profileFile(() -> profileFile);
         return this;
     }
 
     public DefaultServiceEndpointBuilder withProfileName(String profileName) {
-        this.profileName = profileName;
+        endpointResolver.profileName(profileName);
         return this;
     }
 
     public <T> DefaultServiceEndpointBuilder putAdvancedOption(ServiceMetadataAdvancedOption<T> option, T value) {
-        advancedOptions.put(option, value);
+        endpointResolver.putAdvancedOption(option, value);
         return this;
     }
 
     public DefaultServiceEndpointBuilder withDualstackEnabled(Boolean dualstackEnabled) {
-        this.dualstackEnabled = dualstackEnabled;
+        endpointResolver.dualstackEnabled(dualstackEnabled);
         return this;
     }
 
     public DefaultServiceEndpointBuilder withFipsEnabled(Boolean fipsEnabled) {
-        this.fipsEnabled = fipsEnabled;
+        endpointResolver.fipsEnabled(fipsEnabled);
         return this;
     }
 
     public URI getServiceEndpoint() {
-        if (profileFile == null) {
-            profileFile = new Lazy<>(ProfileFile::defaultProfileFile)::getValue;
-        }
-
-        if (profileName == null) {
-            profileName = ProfileFileSystemSetting.AWS_PROFILE.getStringValueOrThrow();
-        }
-
-        if (dualstackEnabled == null) {
-            dualstackEnabled = DualstackEnabledProvider.builder()
-                                                       .profileFile(profileFile)
-                                                       .profileName(profileName)
-                                                       .build()
-                                                       .isDualstackEnabled()
-                                                       .orElse(false);
-        }
-
-        if (fipsEnabled == null) {
-            fipsEnabled = FipsEnabledProvider.builder()
-                                             .profileFile(profileFile)
-                                             .profileName(profileName)
-                                             .build()
-                                             .isFipsEnabled()
-                                             .orElse(false);
-        }
-
-
-
-        List<EndpointTag> endpointTags = new ArrayList<>();
-        if (dualstackEnabled) {
-            endpointTags.add(EndpointTag.DUALSTACK);
-        }
-        if (fipsEnabled) {
-            endpointTags.add(EndpointTag.FIPS);
-        }
-
-        ServiceMetadata serviceMetadata = ServiceMetadata.of(serviceName)
-                                                         .reconfigure(c -> c.profileFile(profileFile)
-                                                                            .profileName(profileName)
-                                                                            .advancedOptions(advancedOptions));
-        URI endpoint = addProtocolToServiceEndpoint(serviceMetadata.endpointFor(ServiceEndpointKey.builder()
-                                                                                                  .region(region)
-                                                                                                  .tags(endpointTags)
-                                                                                                  .build()));
-
-        if (endpoint.getHost() == null) {
-            String error = "Configured region (" + region + ") and tags (" + endpointTags + ") resulted in an invalid URI: "
-                           + endpoint + ". This is usually caused by an invalid region configuration.";
-
-            List<Region> exampleRegions = serviceMetadata.regions();
-            if (!exampleRegions.isEmpty()) {
-                error += " Valid regions: " + exampleRegions;
-            }
-
-            throw SdkClientException.create(error);
-        }
-
-        return endpoint;
-    }
-
-    private URI addProtocolToServiceEndpoint(URI endpointWithoutProtocol) throws IllegalArgumentException {
-        try {
-            return new URI(protocol + "://" + endpointWithoutProtocol);
-        } catch (URISyntaxException e) {
-            throw new IllegalArgumentException(e);
-        }
-    }
-
-    public Region getRegion() {
-        return region;
+        return endpointResolver.build().clientEndpoint();
     }
 }

core/sdk-core/src/main/java/software/amazon/awssdk/core/ClientEndpointProvider.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.core;
+
+import java.net.URI;
+import software.amazon.awssdk.annotations.SdkProtectedApi;
+import software.amazon.awssdk.core.internal.StaticClientEndpointProvider;
+import software.amazon.awssdk.endpoints.EndpointProvider;
+
+/**
+ * Client endpoint providers are responsible for resolving client-level endpoints. {@link EndpointProvider}s are
+ * ultimately responsible for resolving the endpoint used for a request.
+ * <p>
+ * {@link EndpointProvider}s may choose to honor or completely ignore the client-level endpoint. Default endpoint
+ * providers will ignore the client-level endpoint, unless {@link #isEndpointOverridden()} is true.
+ */
+@SdkProtectedApi
+public interface ClientEndpointProvider {
+    /**
+     * Create a client endpoint provider that uses the provided URI and returns true from {@link #isEndpointOverridden()}.
+     */
+    static ClientEndpointProvider forEndpointOverride(URI uri) {
+        return new StaticClientEndpointProvider(uri, true);
+    }
+
+    /**
+     * Create a client endpoint provider that uses the provided static URI and override settings.
+     */
+    static ClientEndpointProvider create(URI uri, boolean isEndpointOverridden) {
+        return new StaticClientEndpointProvider(uri, isEndpointOverridden);
+    }
+
+    /**
+     * Retrieve the client endpoint from this provider.
+     */
+    URI clientEndpoint();
+
+    /**
+     * Returns true if this endpoint was specified as an override by the customer, or false if it was determined
+     * automatically by the SDK.
+     */
+    boolean isEndpointOverridden();
+}

core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkClientOption.java

@@ -23,6 +23,7 @@
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 import software.amazon.awssdk.annotations.SdkProtectedApi;
+import software.amazon.awssdk.core.ClientEndpointProvider;
 import software.amazon.awssdk.core.ClientType;
 import software.amazon.awssdk.core.CompressionConfiguration;
 import software.amazon.awssdk.core.SdkClient;
@@ -100,15 +101,31 @@ public final class SdkClientOption<T> extends ClientOption<T> {
     /**
      * The effective endpoint the client is configured to make requests to. If the client has been configured with
      * an endpoint override then this value will be the provided endpoint value.
+     *
+     * @deprecated Use {@link #CLIENT_ENDPOINT_PROVIDER} for client-level endpoint configuration, or
+     * {@link #ENDPOINT_PROVIDER} for deriving the request-level endpoint.
      */
+    @Deprecated
     public static final SdkClientOption<URI> ENDPOINT = new SdkClientOption<>(URI.class);
 
     /**
      * A flag that when set to true indicates the endpoint stored in {@link SdkClientOption#ENDPOINT} was a customer
      * supplied value and not generated by the client based on Region metadata.
+     *
+     * @deprecated Use {@link #CLIENT_ENDPOINT_PROVIDER}'s {@link ClientEndpointProvider#isEndpointOverridden()}.
      */
+    @Deprecated
     public static final SdkClientOption<Boolean> ENDPOINT_OVERRIDDEN = new SdkClientOption<>(Boolean.class);
 
+    /**
+     * A provider for the client-level endpoint configuration. This includes the default endpoint determined by the
+     * endpoint metadata or endpoint overrides specified by the customer.
+     * <p>
+     * {@link #ENDPOINT_PROVIDER} determines the request-level endpoint configuration.
+     */
+    public static final SdkClientOption<ClientEndpointProvider> CLIENT_ENDPOINT_PROVIDER =
+        new SdkClientOption<>(ClientEndpointProvider.class);
+
     /**
      * Service-specific configuration used by some services, like S3.
      */

core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/ExecutionAttribute.java

@@ -87,6 +87,17 @@ private ExecutionAttribute(String name, ValueStorage<T> storage) {
     public static <T, U> DerivedAttributeBuilder<T, U> derivedBuilder(String name,
                                                                       @SuppressWarnings("unused") Class<T> attributeType,
                                                                       ExecutionAttribute<U> realAttribute) {
+        return new DerivedAttributeBuilder<>(name, () -> realAttribute);
+    }
+
+    /**
+     * This is the same as {@link #derivedBuilder(String, Class, ExecutionAttribute)}, but the real attribute is loaded
+     * lazily at runtime. This is useful when the real attribute is in the same class hierarchy, to avoid initialization
+     * order problems.
+     */
+    public static <T, U> DerivedAttributeBuilder<T, U> derivedBuilder(String name,
+                                                                      @SuppressWarnings("unused") Class<T> attributeType,
+                                                                      Supplier<ExecutionAttribute<U>> realAttribute) {
         return new DerivedAttributeBuilder<>(name, realAttribute);
     }
 
@@ -164,11 +175,11 @@ ValueStorage<T> storage() {
 
     public static final class DerivedAttributeBuilder<T, U> {
         private final String name;
-        private final ExecutionAttribute<U> realAttribute;
+        private final Supplier<ExecutionAttribute<U>> realAttribute;
         private Function<U, T> readMapping;
         private BiFunction<U, T, U> writeMapping;
 
-        private DerivedAttributeBuilder(String name, ExecutionAttribute<U> realAttribute) {
+        private DerivedAttributeBuilder(String name, Supplier<ExecutionAttribute<U>> realAttribute) {
             this.name = name;
             this.realAttribute = realAttribute;
         }
@@ -244,7 +255,7 @@ public void setIfAbsent(Map<ExecutionAttribute<?>, Object> attributes, T value)
      * attributes map.
      */
     private static final class DerivationValueStorage<T, U> implements ValueStorage<T> {
-        private final ExecutionAttribute<U> realAttribute;
+        private final Supplier<ExecutionAttribute<U>> realAttribute;
         private final Function<U, T> readMapping;
         private final BiFunction<U, T, U> writeMapping;
 
@@ -257,13 +268,13 @@ private DerivationValueStorage(DerivedAttributeBuilder<T, U> builder) {
         @SuppressWarnings("unchecked") // Safe because of the implementation of set
         @Override
         public T get(Map<ExecutionAttribute<?>, Object> attributes) {
-            return readMapping.apply((U) attributes.get(realAttribute));
+            return readMapping.apply((U) attributes.get(realAttribute.get()));
         }
 
         @SuppressWarnings("unchecked") // Safe because of the implementation of set
         @Override
         public void set(Map<ExecutionAttribute<?>, Object> attributes, T value) {
-            attributes.compute(realAttribute, (k, real) -> writeMapping.apply((U) real, value));
+            attributes.compute(realAttribute.get(), (k, real) -> writeMapping.apply((U) real, value));
         }
 
         @Override

core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkExecutionAttribute.java

@@ -26,13 +26,16 @@
 import java.util.function.Supplier;
 import software.amazon.awssdk.annotations.SdkPublicApi;
 import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
+import software.amazon.awssdk.core.ClientEndpointProvider;
 import software.amazon.awssdk.core.ClientType;
 import software.amazon.awssdk.core.SelectedAuthScheme;
 import software.amazon.awssdk.core.ServiceConfiguration;
 import software.amazon.awssdk.core.checksums.Algorithm;
 import software.amazon.awssdk.core.checksums.ChecksumSpecs;
 import software.amazon.awssdk.core.checksums.ChecksumValidation;
 import software.amazon.awssdk.core.signer.Signer;
+import software.amazon.awssdk.endpoints.EndpointProvider;
+import software.amazon.awssdk.http.SdkHttpRequest;
 import software.amazon.awssdk.http.auth.aws.signer.AwsV4FamilyHttpSigner;
 import software.amazon.awssdk.http.auth.spi.scheme.AuthSchemeOption;
 import software.amazon.awssdk.http.auth.spi.signer.AsyncSignRequest;
@@ -86,16 +89,38 @@ public class SdkExecutionAttribute {
         new ExecutionAttribute<>("ApiCallAttemptMetricCollector");
 
     /**
-     * If true indicates that the configured endpoint of the client is a value that was supplied as an override and not
+     * True indicates that the configured endpoint of the client is a value that was supplied as an override and not
      * generated from regional metadata.
+     *
+     * @deprecated This value should not be trusted. To modify the endpoint used for requests, you should decorate the
+     * {@link EndpointProvider} of the client. This value can be determined there, by checking for the existence of an
+     * override endpoint.
      */
-    public static final ExecutionAttribute<Boolean> ENDPOINT_OVERRIDDEN = new ExecutionAttribute<>("EndpointOverridden");
+    @Deprecated
+    public static final ExecutionAttribute<Boolean> ENDPOINT_OVERRIDDEN =
+        ExecutionAttribute.derivedBuilder("EndpointOverridden",
+                                          Boolean.class,
+                                          () -> SdkInternalExecutionAttribute.CLIENT_ENDPOINT_PROVIDER)
+                          .readMapping(ClientEndpointProvider::isEndpointOverridden)
+                          .writeMapping((ep, overridden) -> ClientEndpointProvider.create(ep.clientEndpoint(), overridden))
+                          .build();
 
     /**
-     * This is the endpointOverride (if {@link #ENDPOINT_OVERRIDDEN} is true), otherwise the endpoint generated from regional
-     * metadata.
+     * This is the endpointOverride (if {@link #ENDPOINT_OVERRIDDEN} is true), otherwise the endpoint generated from
+     * regional metadata.
+     *
+     * @deprecated This value is not usually accurate, now that the endpoint is almost entirely determined by the
+     * service's endpoint rules. Use {@link SdkHttpRequest#getUri()} from interceptors, to get or modify the actual
+     * endpoint.
      */
-    public static final ExecutionAttribute<URI> CLIENT_ENDPOINT = new ExecutionAttribute<>("EndpointOverride");
+    @Deprecated
+    public static final ExecutionAttribute<URI> CLIENT_ENDPOINT =
+        ExecutionAttribute.derivedBuilder("EndpointOverride",
+                                          URI.class,
+                                          () -> SdkInternalExecutionAttribute.CLIENT_ENDPOINT_PROVIDER)
+                          .readMapping(ClientEndpointProvider::clientEndpoint)
+                          .writeMapping((ep, uri) -> ClientEndpointProvider.create(uri, ep.isEndpointOverridden()))
+                          .build();
 
     /**
      * If the client signer value has been overridden.