diff --git a/.changes/next-release/feature-AWSCRTHTTPClient-69af591.json b/.changes/next-release/feature-AWSCRTHTTPClient-69af591.json new file mode 100644 index 000000000000..a135684797b7 --- /dev/null +++ b/.changes/next-release/feature-AWSCRTHTTPClient-69af591.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "AWS CRT HTTP Client", + "contributor": "", + "description": "Support Non proxy host settings in the ProxyConfiguration for Crt http client." +} diff --git a/core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtConfigurationUtils.java b/core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtConfigurationUtils.java index f7a40b2dc17a..6d31d621a1ca 100644 --- a/core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtConfigurationUtils.java +++ b/core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtConfigurationUtils.java @@ -15,7 +15,11 @@ package software.amazon.awssdk.crtcore; +import static software.amazon.awssdk.utils.StringUtils.lowerCase; + +import java.util.Objects; import java.util.Optional; +import java.util.Set; import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.crt.http.HttpMonitoringOptions; import software.amazon.awssdk.crt.http.HttpProxyOptions; @@ -33,7 +37,9 @@ public static Optional resolveProxy(CrtProxyConfiguration prox if (proxyConfiguration == null) { return Optional.empty(); } - + if (doesTargetMatchNonProxyHosts(proxyConfiguration.host(), proxyConfiguration.nonProxyHosts())) { + return Optional.empty(); + } HttpProxyOptions clientProxyOptions = new HttpProxyOptions(); clientProxyOptions.setHost(proxyConfiguration.host()); @@ -54,11 +60,19 @@ public static Optional resolveProxy(CrtProxyConfiguration prox return Optional.of(clientProxyOptions); } + private static boolean doesTargetMatchNonProxyHosts(String target, Set hostPatterns) { + return Optional.ofNullable(hostPatterns) + .map(patterns -> + patterns.stream() + .filter(Objects::nonNull) + .anyMatch(pattern -> target != null && lowerCase(target).matches(pattern))) + .orElse(false); + } + public static Optional resolveHttpMonitoringOptions(CrtConnectionHealthConfiguration config) { if (config == null) { return Optional.empty(); } - HttpMonitoringOptions httpMonitoringOptions = new HttpMonitoringOptions(); httpMonitoringOptions.setMinThroughputBytesPerSecond(config.minimumThroughputInBps()); int seconds = NumericUtils.saturatedCast(config.minimumThroughputTimeout().getSeconds()); diff --git a/core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtProxyConfiguration.java b/core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtProxyConfiguration.java index 7c15675a5860..42f1dda97975 100644 --- a/core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtProxyConfiguration.java +++ b/core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtProxyConfiguration.java @@ -17,7 +17,10 @@ import static software.amazon.awssdk.utils.ProxyConfigProvider.fromSystemEnvironmentSettings; +import java.util.Collections; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.utils.ProxyConfigProvider; import software.amazon.awssdk.utils.ProxySystemSetting; @@ -36,6 +39,7 @@ public abstract class CrtProxyConfiguration { private final String password; private final Boolean useSystemPropertyValues; private final Boolean useEnvironmentVariableValues; + private final Set nonProxyHosts; protected CrtProxyConfiguration(DefaultBuilder builder) { this.useSystemPropertyValues = builder.useSystemPropertyValues; @@ -49,6 +53,7 @@ protected CrtProxyConfiguration(DefaultBuilder builder) { this.port = resolvePort(builder, proxyConfigProvider); this.username = resolveUsername(builder, proxyConfigProvider); this.password = resolvePassword(builder, proxyConfigProvider); + this.nonProxyHosts = resolveNonProxyHosts(builder, proxyConfigProvider); } private static String resolvePassword(DefaultBuilder builder, ProxyConfigProvider proxyConfigProvider) { @@ -83,6 +88,13 @@ private static String resolveHost(DefaultBuilder builder, ProxyConfigProvider } } + private Set resolveNonProxyHosts(DefaultBuilder builder, ProxyConfigProvider proxyConfigProvider) { + if (builder.nonProxyHosts != null || proxyConfigProvider == null) { + return builder.nonProxyHosts; + } + return proxyConfigProvider.nonProxyHosts(); + } + /** * @return The proxy scheme. */ @@ -132,6 +144,16 @@ public final Boolean isUseEnvironmentVariableValues() { return useEnvironmentVariableValues; } + /** + * Retrieves the hosts that the client is allowed to access without going through the proxy. + * If the value is not set on the object, the value represented by the environment variable or system property is returned. + * + * @see Builder#nonProxyHosts(Set) + */ + public Set nonProxyHosts() { + return Collections.unmodifiableSet(nonProxyHosts != null ? nonProxyHosts : Collections.emptySet()); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -162,7 +184,10 @@ public boolean equals(Object o) { if (!Objects.equals(useSystemPropertyValues, that.useSystemPropertyValues)) { return false; } - return Objects.equals(useEnvironmentVariableValues, that.useEnvironmentVariableValues); + if (!Objects.equals(useEnvironmentVariableValues, that.useEnvironmentVariableValues)) { + return false; + } + return Objects.equals(nonProxyHosts, that.nonProxyHosts); } @Override @@ -175,6 +200,7 @@ public int hashCode() { result = 31 * result + (useSystemPropertyValues != null ? useSystemPropertyValues.hashCode() : 0); result = 31 * result + (useEnvironmentVariableValues != null ? useEnvironmentVariableValues.hashCode() : 0); result = 31 * result + (scheme != null ? scheme.hashCode() : 0); + result = 31 * result + (nonProxyHosts != null ? nonProxyHosts.hashCode() : 0); return result; } @@ -253,6 +279,17 @@ public interface Builder { */ Builder useEnvironmentVariableValues(Boolean useEnvironmentVariableValues); + /** + * Configure the hosts that the client is allowed to access without going through the proxy. + */ + Builder nonProxyHosts(Set nonProxyHosts); + + + /** + * Add a host that the client is allowed to access without going through the proxy. + */ + Builder addNonProxyHost(String nonProxyHost); + CrtProxyConfiguration build(); } @@ -266,6 +303,8 @@ protected abstract static class DefaultBuilder implements Bui private String password; private Boolean useSystemPropertyValues = Boolean.TRUE; private Boolean useEnvironmentVariableValues = Boolean.TRUE; + private Set nonProxyHosts; + protected DefaultBuilder() { } @@ -278,6 +317,7 @@ protected DefaultBuilder(CrtProxyConfiguration proxyConfiguration) { this.port = proxyConfiguration.port; this.username = proxyConfiguration.username; this.password = proxyConfiguration.password; + this.nonProxyHosts = proxyConfiguration.nonProxyHosts; } @Override @@ -322,6 +362,21 @@ public B useEnvironmentVariableValues(Boolean useEnvironmentVariableValues) { return (B) this; } + @Override + public B nonProxyHosts(Set nonProxyHosts) { + this.nonProxyHosts = nonProxyHosts != null ? new HashSet<>(nonProxyHosts) : null; + return (B) this; + } + + @Override + public B addNonProxyHost(String nonProxyHost) { + if (this.nonProxyHosts == null) { + this.nonProxyHosts = new HashSet<>(); + } + this.nonProxyHosts.add(nonProxyHost); + return (B) this; + } + public B setuseEnvironmentVariableValues(Boolean useEnvironmentVariableValues) { return useEnvironmentVariableValues(useEnvironmentVariableValues); } diff --git a/core/crt-core/src/test/java/software/amazon/awssdk/crtcore/CrtConnectionUtilsTest.java b/core/crt-core/src/test/java/software/amazon/awssdk/crtcore/CrtConnectionUtilsTest.java index 2c68f6b4cefc..2c6bc24cc48a 100644 --- a/core/crt-core/src/test/java/software/amazon/awssdk/crtcore/CrtConnectionUtilsTest.java +++ b/core/crt-core/src/test/java/software/amazon/awssdk/crtcore/CrtConnectionUtilsTest.java @@ -20,7 +20,11 @@ import java.time.Duration; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; import software.amazon.awssdk.crt.http.HttpMonitoringOptions; import software.amazon.awssdk.crt.http.HttpProxyOptions; @@ -54,6 +58,60 @@ void resolveProxy_emptyProxy_shouldReturnEmpty() { assertThat(CrtConfigurationUtils.resolveProxy(null, tlsContext)).isEmpty(); } + @ParameterizedTest + @ValueSource(strings = {".*?.2.3.4", "1.*?.3.4", ".*?"}) + void resolveProxy_withSingleNonProxyHostsWidCards_shouldReturnEmpty(String nonProxyHost) { + TlsContext tlsContext = Mockito.mock(TlsContext.class); + CrtProxyConfiguration configuration = new TestProxy.Builder().host("1.2.3.4") + .port(123) + .scheme("https") + .password("bar") + .username("foo") + .nonProxyHosts(Stream.of(nonProxyHost,"someRandom") + .collect(Collectors.toSet())) + .build(); + assertThat(CrtConfigurationUtils.resolveProxy(configuration, tlsContext)).isEmpty(); + } + + + + @Test + void resolveProxy_withNullHostAndNonPorxy_shouldNotReturnEmpty( ) { + TlsContext tlsContext = Mockito.mock(TlsContext.class); + CrtProxyConfiguration configuration = new TestProxy.Builder().host(null) + .port(123) + .scheme("https") + .password("bar") + .username("foo") + .nonProxyHosts(Stream.of("someRandom", "null") + .collect(Collectors.toSet())) + .build(); + assertThat(CrtConfigurationUtils.resolveProxy(configuration, tlsContext)).isNotEmpty(); + } + + @Test + void resolveProxy_basicAuthorization_WithNonMatchingNoProxy() { + CrtProxyConfiguration configuration = new TestProxy.Builder().host("1.2.3.4") + .port(123) + .scheme("https") + .password("bar") + .addNonProxyHost("someRandom") + .addNonProxyHost(null) + .username("foo") + .build(); + + TlsContext tlsContext = Mockito.mock(TlsContext.class); + + Optional httpProxyOptions = CrtConfigurationUtils.resolveProxy(configuration, tlsContext); + assertThat(httpProxyOptions).hasValueSatisfying(proxy -> { + assertThat(proxy.getTlsContext()).isEqualTo(tlsContext); + assertThat(proxy.getAuthorizationPassword()).isEqualTo("bar"); + assertThat(proxy.getAuthorizationUsername()).isEqualTo("foo"); + assertThat(proxy.getAuthorizationType()).isEqualTo(HttpProxyOptions.HttpProxyAuthorizationType.Basic); + }); + } + + @Test void resolveProxy_noneAuthorization() { CrtProxyConfiguration configuration = new TestProxy.Builder().host("1.2.3.4") diff --git a/http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/ProxyConfiguration.java b/http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/ProxyConfiguration.java index b50036fdcb5c..6e6cd85dd775 100644 --- a/http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/ProxyConfiguration.java +++ b/http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/ProxyConfiguration.java @@ -15,8 +15,10 @@ package software.amazon.awssdk.http.crt; +import java.util.Set; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.crtcore.CrtProxyConfiguration; +import software.amazon.awssdk.utils.ProxyEnvironmentSetting; import software.amazon.awssdk.utils.ProxySystemSetting; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -116,9 +118,35 @@ public interface Builder extends CrtProxyConfiguration.Builder, CopyableBuilder< Builder useSystemPropertyValues(Boolean useSystemPropertyValues); + /** + * Set the option whether to use environment variable values for {@link ProxyEnvironmentSetting} if any of the config + * options are missing. The value is set to "true" by default, enabling the SDK to automatically use environment variable + * values for proxy configuration options that are not provided during building the {@link ProxyConfiguration} object. To + * disable this behavior, set this value to "false".It is important to note that when this property is set to "true," all + * proxy settings will exclusively originate from Environment Variable Values, and no partial settings will be obtained + * from System Property Values. + *

Comma-separated host names in the NO_PROXY environment variable indicate multiple hosts to exclude from + * proxy settings. + * + * @param useEnvironmentVariableValues The option whether to use environment variable values + * @return This object for method chaining. + */ @Override Builder useEnvironmentVariableValues(Boolean useEnvironmentVariableValues); + /** + * Configure the hosts that the client is allowed to access without going through the proxy. + */ + @Override + Builder nonProxyHosts(Set nonProxyHosts); + + + /** + * Add a host that the client is allowed to access without going through the proxy. + */ + @Override + Builder addNonProxyHost(String nonProxyHost); + @Override ProxyConfiguration build(); } diff --git a/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/CrtHttpProxyTest.java b/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/CrtHttpProxyTest.java index 4095302dd8f3..ae2f66aca1e3 100644 --- a/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/CrtHttpProxyTest.java +++ b/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/CrtHttpProxyTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.net.URISyntaxException; +import java.util.Set; import software.amazon.awssdk.http.HttpProxyTestSuite; import software.amazon.awssdk.http.proxy.TestProxySetting; @@ -35,6 +36,7 @@ protected void assertProxyConfiguration(TestProxySetting userSetProxySettings, Integer portNumber = userSetProxySettings.getPort(); String userName = userSetProxySettings.getUserName(); String password = userSetProxySettings.getPassword(); + Set nonProxyHosts = userSetProxySettings.getNonProxyHosts(); if (hostName != null) { proxyBuilder.host(hostName); @@ -48,6 +50,9 @@ protected void assertProxyConfiguration(TestProxySetting userSetProxySettings, if (password != null) { proxyBuilder.password(password); } + if (nonProxyHosts != null && !nonProxyHosts.isEmpty()) { + proxyBuilder.nonProxyHosts(nonProxyHosts); + } } if (!"http".equals(protocol)) { @@ -64,6 +69,7 @@ protected void assertProxyConfiguration(TestProxySetting userSetProxySettings, assertThat(proxyConfiguration.port()).isEqualTo(expectedProxySettings.getPort()); assertThat(proxyConfiguration.username()).isEqualTo(expectedProxySettings.getUserName()); assertThat(proxyConfiguration.password()).isEqualTo(expectedProxySettings.getPassword()); + assertThat(proxyConfiguration.nonProxyHosts()).isEqualTo(expectedProxySettings.getNonProxyHosts()); } } diff --git a/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyConfigurationTest.java b/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyConfigurationTest.java index 9d4d25f7b47c..bd535d23358b 100644 --- a/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyConfigurationTest.java +++ b/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyConfigurationTest.java @@ -19,7 +19,11 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.HashSet; import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; @@ -181,6 +185,8 @@ private void setRandomValue(Object o, Method setter) throws InvocationTargetExce setter.invoke(o, RNG.nextInt()); } else if (Boolean.class.equals(paramClass)) { setter.invoke(o, RNG.nextBoolean()); + } else if (Set.class.equals(paramClass)) { + setter.invoke(o, IntStream.range(0, 5).mapToObj(i -> randomString()).collect(Collectors.toSet())); } else { throw new RuntimeException("Don't know how create random value for type " + paramClass); } diff --git a/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyWireMockTest.java b/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyWireMockTest.java index 7c56119bc26e..47fb56d70fba 100644 --- a/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyWireMockTest.java +++ b/http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyWireMockTest.java @@ -26,9 +26,12 @@ import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import java.net.URI; import java.nio.ByteBuffer; +import java.security.SecureRandom; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -68,21 +71,16 @@ public void setup() { mockServer.stubFor(get(urlMatching(".*")).willReturn(aResponse().withStatus(200).withBody("hello"))); - proxyCfg = ProxyConfiguration.builder() - .host("localhost") - .port(mockProxy.port()) - .build(); - client = AwsCrtAsyncHttpClient.builder() - .proxyConfiguration(proxyCfg) - .build(); } @AfterEach public void teardown() { mockServer.stop(); mockProxy.stop(); - client.close(); + if(client != null){ + client.close(); + } EventLoopGroup.closeStaticDefault(); HostResolver.closeStaticDefault(); CrtResource.waitForNoResources(); @@ -99,6 +97,15 @@ public void teardown() { @Test public void proxyConfigured_httpGet() throws Throwable { + proxyCfg = ProxyConfiguration.builder() + .host("localhost") + .port(mockProxy.port()) + .build(); + + client = AwsCrtAsyncHttpClient.builder() + .proxyConfiguration(proxyCfg) + .build(); + CompletableFuture streamReceived = new CompletableFuture<>(); AtomicReference response = new AtomicReference<>(null); AtomicReference error = new AtomicReference<>(null); @@ -121,4 +128,47 @@ public void proxyConfigured_httpGet() throws Throwable { assertThat(response.get().statusCode()).isEqualTo(200); } + @Test + public void proxyConfiguredAndSkippedUsingNonProxy_httpGet() throws Throwable { + SecureRandom secureRandom = new SecureRandom(); + int randomPort = 0; + // Generate a random port to test if the proxy host is not routed to when nonProxyHosts is specified. + // If the nonProxyHosts had not matched, a connection exception would have occurred. + do { + randomPort = secureRandom.nextInt(65535); + } while (randomPort == mockProxy.port()); + + proxyCfg = ProxyConfiguration.builder() + .host("localhost") + .port(randomPort) + .nonProxyHosts(Stream.of("localhost").collect(Collectors.toSet())) + .build(); + + client = AwsCrtAsyncHttpClient.builder() + .proxyConfiguration(proxyCfg) + .build(); + + CompletableFuture streamReceived = new CompletableFuture<>(); + AtomicReference response = new AtomicReference<>(null); + AtomicReference error = new AtomicReference<>(null); + + Subscriber subscriber = CrtHttpClientTestUtils.createDummySubscriber(); + + SdkAsyncHttpResponseHandler handler = CrtHttpClientTestUtils.createTestResponseHandler(response, streamReceived, error, + subscriber); + + URI uri = URI.create("http://localhost:" + mockServer.port()); + SdkHttpRequest request = CrtHttpClientTestUtils.createRequest(uri, "/server/test", null, SdkHttpMethod.GET, emptyMap()); + + CompletableFuture future = client.execute(AsyncExecuteRequest.builder() + .request(request) + .responseHandler(handler) + .requestContentPublisher(new EmptyPublisher()) + .build()); + future.get(60, TimeUnit.SECONDS); + assertThat(error.get()).isNull(); + assertThat(streamReceived.get(60, TimeUnit.SECONDS)).isTrue(); + assertThat(response.get().statusCode()).isEqualTo(200); + } + }